[proposal] Internal mutateArgs usage in core

lib/waterline/MetaModel.js

@@ -157,6 +157,7 @@ _.extend(
     createEach: require('./methods/create-each'),
     update: require('./methods/update'),
     destroy: require('./methods/destroy'),
+    archive: require('./methods/archive'),
     addToCollection: require('./methods/add-to-collection'),
     removeFromCollection: require('./methods/remove-from-collection'),
     replaceCollection: require('./methods/replace-collection'),

lib/waterline/methods/add-to-collection.js

@@ -296,9 +296,10 @@ module.exports = function addToCollection(/* targetRecordIds, collectionAttrName
           manyToMany = true;
         }
 
-        // Ensure the query skips lifecycle callbacks
         // Build a modified shallow clone of the originally-provided `meta`
-        var modifiedMeta = _.extend({}, query.meta || {}, { skipAllLifecycleCallbacks: true });
+        // To ensure that internal queries skip lifecycle callbacks, and that they don't worry
+        // about protecting arguments from destructive mutation.
+        var modifiedMeta = _.extend({}, query.meta || {}, { skipAllLifecycleCallbacks: true, mutateArgs: true });
 
 
         //  ███╗   ███╗ █████╗ ███╗   ██╗██╗   ██╗    ████████╗ ██████╗     ███╗   ███╗ █████╗ ███╗   ██╗██╗   ██╗

lib/waterline/methods/create-each.js

@@ -397,14 +397,19 @@ module.exports = function createEach( /* newRecords?, explicitCbMaybe?, meta? */
             });// </ each key in "reset" >
           });//</ each record>
 
+          // In our `meta` query key for use in these internal queries, enable the
+          // `mutateArgs` flag to increase performance (by avoiding the unnecessary
+          // duplication of objects and allowing arguments to be destructively consumed).
+          var metaForInternalQueries = _.extend({}, query.meta||{}, { mutateArgs: true });
+
           async.each(argsForEachReplaceOp, function _eachReplaceCollectionOp(argsForReplace, next) {
 
             // Note that, by using the same `meta`, we use same db connection
             // (if one was explicitly passed in, anyway)
             WLModel.replaceCollection(argsForReplace[0], argsForReplace[1], argsForReplace[2], function(err) {
               if (err) { return next(err); }
               return next();
-            }, query.meta);
+            }, metaForInternalQueries);
 
           },// ~∞%°
           function _afterReplacingAllCollections(err) {

lib/waterline/methods/create.js

@@ -331,12 +331,18 @@ module.exports = function create(newRecord, explicitCbMaybe, metaContainer) {
           //  ├┤ ┌┴┬┘├─┘│  ││  │ │ │ └┬┘───└─┐├─┘├┤ │  │├┤ │├┤  ││  ├─┤└─┐└─┐│ ││  │├─┤ │ ││ ││││└─┐
           //  └─┘┴ └─┴  ┴─┘┴└─┘┴ ┴ ┴─┘┴    └─┘┴  └─┘└─┘┴└  ┴└─┘─┴┘  ┴ ┴└─┘└─┘└─┘└─┘┴┴ ┴ ┴ ┴└─┘┘└┘└─┘
           var targetId = transformedRecord[WLModel.primaryKey];
+
+          // In our `meta` query key for use in these internal queries, enable the
+          // `mutateArgs` flag to increase performance (by avoiding the unnecessary
+          // duplication of objects and allowing arguments to be destructively consumed).
+          var metaForInternalQueries = _.extend({}, query.meta||{}, { mutateArgs: true });
+
           async.each(_.keys(collectionResets), function _eachReplaceCollectionOp(collectionAttrName, next) {
 
             WLModel.replaceCollection(targetId, collectionAttrName, collectionResets[collectionAttrName], function(err){
               if (err) { return next(err); }
               return next();
-            }, query.meta);
+            }, metaForInternalQueries);
 
           },// ~∞%°
           function _afterReplacingAllCollections(err) {

lib/waterline/methods/destroy.js

@@ -395,6 +395,11 @@ module.exports = function destroy(/* criteria, explicitCbMaybe, metaContainer */
               // (if n..m, this destroys junction records; otherwise, it's n..1, so this just nulls out the other side)
               //
               // > Note that we pass through `meta` here, ensuring that the same db connection is used, if possible.
+              // > But in our `meta` query key for use in these internal queries, we enable the
+              // > `mutateArgs` flag to increase performance (by avoiding the unnecessary
+              // > duplication of objects and allowing arguments to be destructively consumed).
+              var metaForInternalQueries = _.extend({}, query.meta||{}, { mutateArgs: true });
+
               async.each(_.keys(WLModel.attributes), function _eachAttribute(attrName, next) {
 
                 var attrDef = WLModel.attributes[attrName];
@@ -424,7 +429,7 @@ module.exports = function destroy(/* criteria, explicitCbMaybe, metaContainer */
 
                   return next();
 
-                }, query.meta);//</.replaceCollection()>
+                }, metaForInternalQueries);//</.replaceCollection()>
 
               },// ~∞%°
               function _afterwards(err) {
@@ -546,12 +551,12 @@ module.exports = function destroy(/* criteria, explicitCbMaybe, metaContainer */
                   }
 
                   return done(undefined, transformedRecordsMaybe);
-                });//</ async.each() -- ran "after" lifecycle callback on each record >
-              });//</ after determining (and potentially transforming) the result from the adapter >
-            }); // </ _afterPotentiallyWipingCollections >
-          }); // </ adapter.destroy >
-        }); // </ afterPotentiallyLookingUpRecordsToCascade >
-      }); // </ _afterRunningBeforeLC >
+                });//_∏_   (†: async.each() -- ran "after" lifecycle callback on each record)
+              });//_∏_   (†: after determining (and potentially transforming) the result from the adapter)
+            });//_∏_   (†: _afterPotentiallyWipingCollections)
+          });//_∏_   (adapter.destroy)
+        }); //_∏_   (†: after potentially looking up records to cascade)
+      }); //_∏_   (†: "before" LC)
     },
 
 

lib/waterline/methods/find-or-create.js

@@ -208,6 +208,11 @@ module.exports = function findOrCreate( /* criteria?, newRecord?, explicitCbMayb
       delete query.criteria.skip;
       delete query.criteria.sort;
 
+      // In our `meta` query key for use in these internal queries, enable the
+      // `mutateArgs` flag to increase performance (by avoiding the unnecessary
+      // duplication of objects and allowing arguments to be destructively consumed).
+      var metaForInternalQueries = _.extend({}, query.meta||{}, { mutateArgs: true });
+
 
       //  ╔═╗═╗ ╦╔═╗╔═╗╦ ╦╔╦╗╔═╗  ┌─┐┬┌┐┌┌┬┐  ┌─┐┌┐┌┌─┐  ┌─┐ ┬ ┬┌─┐┬─┐┬ ┬
       //  ║╣ ╔╩╦╝║╣ ║  ║ ║ ║ ║╣   ├┤ ││││ ││  │ ││││├┤   │─┼┐│ │├┤ ├┬┘└┬┘
@@ -240,7 +245,7 @@ module.exports = function findOrCreate( /* criteria?, newRecord?, explicitCbMayb
 
         // Build a modified shallow clone of the originally-provided `meta`
         // that also has `fetch: true`.
-        var modifiedMeta = _.extend({}, query.meta || {}, { fetch: true });
+        var modifiedMeta = _.extend({}, metaForInternalQueries, { fetch: true });
 
         //  ╔═╗═╗ ╦╔═╗╔═╗╦ ╦╔╦╗╔═╗  ┌─┐┬─┐┌─┐┌─┐┌┬┐┌─┐  ┌─┐ ┬ ┬┌─┐┬─┐┬ ┬
         //  ║╣ ╔╩╦╝║╣ ║  ║ ║ ║ ║╣   │  ├┬┘├┤ ├─┤ │ ├┤   │─┼┐│ │├┤ ├┬┘└┬┘
@@ -261,7 +266,7 @@ module.exports = function findOrCreate( /* criteria?, newRecord?, explicitCbMayb
           return done(undefined, createdRecord, true);
 
         }, modifiedMeta);//</.create()>
-      }, query.meta);//</.findOne()>
+      }, metaForInternalQueries);//</.findOne()>
     },
 
 

lib/waterline/methods/remove-from-collection.js

@@ -299,9 +299,10 @@ module.exports = function removeFromCollection(/* targetRecordIds?, collectionAt
           manyToMany = true;
         }
 
-        // Ensure the query skips lifecycle callbacks
         // Build a modified shallow clone of the originally-provided `meta`
-        var modifiedMeta = _.extend({}, query.meta || {}, { skipAllLifecycleCallbacks: true });
+        // To ensure that internal queries skip lifecycle callbacks, and that they don't worry
+        // about protecting arguments from destructive mutation.
+        var modifiedMeta = _.extend({}, query.meta || {}, { skipAllLifecycleCallbacks: true, mutateArgs: true });
 
 
         //   ██╗███╗   ██╗      ███╗   ███╗██╗

lib/waterline/methods/replace-collection.js

@@ -297,9 +297,10 @@ module.exports = function replaceCollection(/* targetRecordIds?, collectionAttrN
         }
 
 
-        // Ensure the query skips lifecycle callbacks
         // Build a modified shallow clone of the originally-provided `meta`
-        var modifiedMeta = _.extend({}, query.meta || {}, { skipAllLifecycleCallbacks: true });
+        // To ensure that internal queries skip lifecycle callbacks, and that they don't worry
+        // about protecting arguments from destructive mutation.
+        var modifiedMeta = _.extend({}, query.meta || {}, { skipAllLifecycleCallbacks: true, mutateArgs: true });
 
 
 

lib/waterline/methods/stream.js

@@ -261,6 +261,11 @@ module.exports = function stream( /* criteria?, eachRecordFn?, explicitCbMaybe?,
       var i = 0;
 
 
+      // In our `meta` query key for use in these internal queries, enable the
+      // `mutateArgs` flag to increase performance (by avoiding the unnecessary
+      // duplication of objects and allowing arguments to be destructively consumed).
+      var metaForInternalQueries = _.extend({}, query.meta||{}, { mutateArgs: true });
+
       async.whilst(function _checkHasntReachedLastBatchYet(){
         if (!reachedLastBatch) { return true; }
         else { return false; }
@@ -318,7 +323,7 @@ module.exports = function stream( /* criteria?, eachRecordFn?, explicitCbMaybe?,
 
         // Pass through `meta` so we're sure to use the same db connection
         // and settings (i.e. esp. relevant if we happen to be inside a transaction)
-        deferredForThisBatch.meta(query.meta);
+        deferredForThisBatch.meta(metaForInternalQueries);
 
         deferredForThisBatch.exec(function (err, batchOfRecords){
           if (err) { return next(err); }

lib/waterline/methods/update.js

@@ -382,12 +382,18 @@ module.exports = function update(criteria, valuesToSet, explicitCbMaybe, metaCon
           //  ├┤ ┌┴┬┘├─┘│  ││  │ │ │ └┬┘───└─┐├─┘├┤ │  │├┤ │├┤  ││  ├─┤└─┐└─┐│ ││  │├─┤ │ ││ ││││└─┐
           //  └─┘┴ └─┴  ┴─┘┴└─┘┴ ┴ ┴─┘┴    └─┘┴  └─┘└─┘┴└  ┴└─┘─┴┘  ┴ ┴└─┘└─┘└─┘└─┘┴┴ ┴ ┴ ┴└─┘┘└┘└─┘
           var targetIds = _.pluck(transformedRecords, WLModel.primaryKey);
+
+          // In our `meta` query key for use in these internal queries, enable the
+          // `mutateArgs` flag to increase performance (by avoiding the unnecessary
+          // duplication of objects and allowing arguments to be destructively consumed).
+          var metaForInternalQueries = _.extend({}, query.meta||{}, { mutateArgs: true });
+
           async.each(_.keys(collectionResets), function _eachReplaceCollectionOp(collectionAttrName, next) {
 
             WLModel.replaceCollection(targetIds, collectionAttrName, collectionResets[collectionAttrName], function(err){
               if (err) { return next(err); }
               return next();
-            }, query.meta);
+            }, metaForInternalQueries);
 
           },// ~∞%°
           function _afterReplacingAllCollections(err) {

lib/waterline/utils/query/forge-stage-two-query.js

@@ -178,6 +178,7 @@ module.exports = function forgeStageTwoQuery(query, orm) {
 
       case 'update':               return [ 'criteria', 'valuesToSet' ];
       case 'destroy':              return [ 'criteria' ];
+      case 'archive':              return [ 'criteria' ];
       case 'addToCollection':      return [ 'targetRecordIds', 'collectionAttrName', 'associatedIds' ];
       case 'removeFromCollection': return [ 'targetRecordIds', 'collectionAttrName', 'associatedIds' ];
       case 'replaceCollection':    return [ 'targetRecordIds', 'collectionAttrName', 'associatedIds' ];
@@ -412,6 +413,12 @@ module.exports = function forgeStageTwoQuery(query, orm) {
     //  ┌┬┐┬ ┬┌┬┐┌─┐┌┬┐┌─┐  ┌─┐┬─┐┌─┐┌─┐
     //  ││││ │ │ ├─┤ │ ├┤   ├─┤├┬┘│ ┬└─┐
     //  ┴ ┴└─┘ ┴ ┴ ┴ ┴ └─┘  ┴ ┴┴└─└─┘└─┘
+    //
+    // EXPERIMENTAL: The `mutateArgs` meta key enabled optimizations by preventing
+    // unnecessary cloning of arguments.
+    //
+    // > Note that this is ONLY respected at the stage 2 level!
+    // > That is, it doesn't matter if this meta key is set or not when you call adapters.
     if (query.meta.mutateArgs !== undefined) {
 
       if (!_.isBoolean(query.meta.mutateArgs)) {
@@ -508,9 +515,9 @@ module.exports = function forgeStageTwoQuery(query, orm) {
     }//>-•
 
     // If the criteria is not defined, then in most cases, we treat it like `{}`.
-    // BUT if this query will be running as a result of an `update()` or a `destroy()`,
-    // then we'll be a bit more picky in order to prevent accidents.
-    if (_.isUndefined(query.criteria) && (query.method === 'update' || query.method === 'destroy')) {
+    // BUT if this query will be running as a result of an `update()`, or a `destroy()`,
+    // or an `.archive()`, then we'll be a bit more picky in order to prevent accidents.
+    if (_.isUndefined(query.criteria) && (query.method === 'update' || query.method === 'destroy' || query.method === 'archive')) {
 
       throw buildUsageError('E_INVALID_CRITERIA', 'Cannot use this method (`'+query.method+'`) with a criteria of `undefined`.  (This is just a simple failsafe to help protect your data: if you really want to '+query.method+' ALL records, no problem-- please just be explicit and provide a criteria of `{}`.)', query.using);
 

lib/waterline/utils/query/get-query-modifier-methods.js

@@ -688,6 +688,8 @@ module.exports = function getQueryModifierMethods(category){
 
     case 'update':               _.extend(queryMethods, FILTER_Q_METHODS, SET_Q_METHODS, FETCH_Q_METHODS); break;
     case 'destroy':              _.extend(queryMethods, FILTER_Q_METHODS, FETCH_Q_METHODS); break;
+    case 'archive':              _.extend(queryMethods, FILTER_Q_METHODS, FETCH_Q_METHODS); break;
+
     case 'addToCollection':      _.extend(queryMethods, COLLECTION_Q_METHODS); break;
     case 'removeFromCollection': _.extend(queryMethods, COLLECTION_Q_METHODS); break;
     case 'replaceCollection':    _.extend(queryMethods, COLLECTION_Q_METHODS); break;

lib/waterline/utils/query/private/normalize-where-clause.js

@@ -85,7 +85,36 @@ module.exports = function normalizeWhereClause(whereClause, modelIdentity, orm,
     // (This isn't great because it can mess up things like Buffers... which you
     // shouldn't really be using in a `where` clause anyway, but still-- it makes
     // it way harder to figure out what's wrong when folks find themselves in that
-    // situation.)
+    // situation.  It could also affect any weird custom constraints for `type:'ref'`
+    // attrs.  And if the current approach were also used in valuesToSet, newRecord,
+    // newRecords etc, it would matter even more.)
+    //
+    // The full list of query keys that need to be carefully checked:
+    // • criteria
+    // • populates
+    // • newRecord
+    // • newRecords
+    // • valuesToSet
+    // • targetRecordIds
+    // • associatedIds
+    //
+    // The solution will probably mean distributing this deep clone behavior out
+    // to the various places it's liable to come up.  In reality, this will be
+    // more performant anyway, since we won't be unnecessarily cloning things like
+    // big JSON values, etc.
+    //
+    // The important thing is that this should do shallow clones of deeply-nested
+    // control structures like top level query key dictionaries, criteria clauses,
+    // predicates/constraints/modifiers in `where` clauses, etc.
+    //
+    // > And remember:  Don't deep-clone functions.
+    // > Note that, weirdly, it's fine to deep-clone dictionaries/arrays
+    // > that contain nested functions (they just don't get cloned-- they're
+    // > the same reference).  But if you try to deep-clone a function at the
+    // > top level, it gets messed up.
+    // >
+    // > More background on this: https://trello.com/c/VLXPvyN5
+    // > (Note that this behavior maintains backwards compatibility with Waterline <=0.12.)
     // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
   }//fi