From 08e15bdfe04a3767772077d96dca9bccf5b4c525 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Tue, 3 Sep 2019 17:12:26 +0200 Subject: [PATCH 1/6] First draft complex process graph reducers. --- src/processes/first.js | 6 ++- src/processes/last.js | 6 ++- src/processes/max.js | 52 +++++++++++++++++++- src/processes/mean.js | 6 ++- src/processes/median.js | 6 ++- src/processes/min.js | 6 ++- src/processes/reduce.js | 83 ++++++++++++++++++++++++++++---- src/processes/sd.js | 6 ++- src/processes/sum.js | 6 ++- src/processes/variance.js | 6 ++- src/processgraph/commons.js | 27 ----------- src/processgraph/processgraph.js | 6 +++ 12 files changed, 171 insertions(+), 45 deletions(-) diff --git a/src/processes/first.js b/src/processes/first.js index bf7e14d..f469d39 100644 --- a/src/processes/first.js +++ b/src/processes/first.js @@ -3,8 +3,12 @@ const Commons = require('../processgraph/commons'); module.exports = class first extends Process { + geeReducer() { + return 'first'; + } + async execute(node, context) { - return Commons.reduceInCallback(node, 'first'); + return node.getData("data"); } }; \ No newline at end of file diff --git a/src/processes/last.js b/src/processes/last.js index 6c59c19..e341aff 100644 --- a/src/processes/last.js +++ b/src/processes/last.js @@ -3,8 +3,12 @@ const Commons = require('../processgraph/commons'); module.exports = class last extends Process { + geeReducer() { + return 'last'; + } + async execute(node, context) { - return Commons.reduceInCallback(node, 'last'); + return node.getData("data"); } }; \ No newline at end of file diff --git a/src/processes/max.js b/src/processes/max.js index 6acd969..87abb79 100644 --- a/src/processes/max.js +++ b/src/processes/max.js @@ -3,8 +3,58 @@ const Commons = require('../processgraph/commons'); module.exports = class max extends Process { + geeReducer() { + return 'max'; + } + async execute(node, context) { - return Commons.reduceInCallback(node, 'max'); + var list = node.getArgument('data'); + if (list.length <= 1) { + // ToDo: Invalid, handle appropriately + return; + } + + for(var i = 1; i < list.length; i++) { + var first = list[i-1]; + var second = list[i]; + if (typeof first === 'number' && typeof second === 'number') { + list[i] = Math.max(first, second); + } + else if (first.isImageCollection()) { + if (typeof second === 'number') { + list[i] = first.imageCollection(ic => ic.map(img => img.max(second))); + } + else if (second.isImage()) { + list[i] = second.image(img => img.max(first)); + } + else { + throw "Not supported"; + } + } + else if (first.isImage()) { + if (typeof second === 'number' || second.isImage()) { + list[i] = first.image(img => img.max(second)); + } + else if (second.isImageCollection()) { + list[i] = second.imageCollection(ic => ic.map(img => img.max(first))); + } + else { + throw "Not supported"; + } + } + else if (first.isArray()) { + if (typeof second === 'number' || second.isArray()) { + list[i] = first.array(arr => arr.max(second)); // Does this work for numbers? + } + else { + throw "Not supported"; + } + } + else { + throw "Not supported"; + } + } + return list[list.length-1]; } }; \ No newline at end of file diff --git a/src/processes/mean.js b/src/processes/mean.js index 66ce13f..c9ddb3e 100644 --- a/src/processes/mean.js +++ b/src/processes/mean.js @@ -3,8 +3,12 @@ const Commons = require('../processgraph/commons'); module.exports = class mean extends Process { + geeReducer() { + return 'mean'; + } + async execute(node, context) { - return Commons.reduceInCallback(node, 'mean'); + return node.getData("data"); } }; \ No newline at end of file diff --git a/src/processes/median.js b/src/processes/median.js index 717b659..2e464bd 100644 --- a/src/processes/median.js +++ b/src/processes/median.js @@ -3,8 +3,12 @@ const Commons = require('../processgraph/commons'); module.exports = class median extends Process { + geeReducer() { + return 'median'; + } + async execute(node, context) { - return Commons.reduceInCallback(node, 'median'); + return node.getData("data"); } }; \ No newline at end of file diff --git a/src/processes/min.js b/src/processes/min.js index fe39cde..9342132 100644 --- a/src/processes/min.js +++ b/src/processes/min.js @@ -3,8 +3,12 @@ const Commons = require('../processgraph/commons'); module.exports = class min extends Process { + geeReducer() { + return 'min'; + } + async execute(node, context) { - return Commons.reduceInCallback(node, 'min'); + return node.getData("data"); } }; \ No newline at end of file diff --git a/src/processes/reduce.js b/src/processes/reduce.js index 66d7d8e..b9fac52 100644 --- a/src/processes/reduce.js +++ b/src/processes/reduce.js @@ -6,22 +6,87 @@ module.exports = class reduce extends Process { async execute(node, context) { var dc = node.getData("data"); - var dimension = node.getArgument("dimension"); - var temporalDimension = dc.getDimension(dimension); - if (temporalDimension.type !== 'temporal') { + + var dimensionName = node.getArgument("dimension"); + var dimension = dc.getDimension(dimensionName); + if (dimension.type !== 'temporal' && dimension.type !== 'bands') { throw new Errors.ProcessArgumentInvalid({ process: this.schema.id, argument: 'dimension', - reason: 'GEE can only reduce the temporal dimension at the moment.' + reason: 'Reducing dimension types other than `temporal` or `bands` is currently not supported.' }); } + + var resultDataCube; var callback = node.getArgument("reducer"); - var resultNode = await callback.execute({ - data: dc - }); - dc = resultNode.getResult(); - dc.dropDimension(dimension); + if (callback.getNodeCount() === 1) { + // This is a simple reducer with just one node + var process = callback.getProcess(callback.getResultNode()); + if (typeof process.geeReducer !== 'function') { + throw new Errors.ProcessArgumentInvalid({ + process: this.schema.id, + argument: 'reducer', + reason: 'The specified reducer is invalid.' + }); + } + resultDataCube = this.reduceSimple(dc, process.geeReducer()); + } + else { + var values; + if (dimension.type === 'temporal') { + var ic = data.imageCollection(); + values = ic.toList(); + } + else if (dimension.type === 'bands') { + var ic = data.imageCollection(); + // ToDo: Ensure that the bands have a fixed order! + values = data.getBands().map(band => ic.select(band)); + } + // This is a complex reducer +// var resultNode = this.reduceComplex(); + var resultNode = await callback.execute({ + data: ic, + values: values + }); + resultDataCube = resultNode.getResult(); + } + resultDataCube.dropDimension(dimensionName); + return resultDataCube; + } + + reduceSimple(dc, reducerFunc, reducerName = null) { + if (reducerName === null) { + if (typeof reducerFunc === 'string') { + reducerName = reducerFunc; + } + else { + throw new Error("The parameter 'reducerName' must be specified."); + } + } + + var bands = dc.getBands(); + var renamedBands = bands.map(bandName => bandName + "_" + reducerName); + if (dc.isImageCollection()) { + dc.imageCollection(data => data.reduce(reducerFunc).map( + // revert renaming of the bands following to the GEE convention + image => image.select(renamedBands).rename(bands) + )); + } + else if (dc.isImage()) { + // reduce and revert renaming of the bands following to the GEE convention + dc.image(img => img.reduce(reducerFunc).select(renamedBands).rename(bands)); + } + else if (dc.isArray()) { + dc.array(data => data.reduce(reducerFunc)); + } + else { + throw new Error("Calculating " + reducerName + " not supported for given data type."); + } return dc; } + reduceComplex() { + + } + }; \ No newline at end of file diff --git a/src/processes/sd.js b/src/processes/sd.js index 875f479..7a1882e 100644 --- a/src/processes/sd.js +++ b/src/processes/sd.js @@ -3,8 +3,12 @@ const Commons = require('../processgraph/commons'); module.exports = class sd extends Process { + geeReducer() { + return 'stdDev'; + } + async execute(node, context) { - return Commons.reduceInCallback(node, 'stdDev'); + return node.getData("data"); } }; \ No newline at end of file diff --git a/src/processes/sum.js b/src/processes/sum.js index 8bb6b04..ef7c2e5 100644 --- a/src/processes/sum.js +++ b/src/processes/sum.js @@ -3,8 +3,12 @@ const Commons = require('../processgraph/commons'); module.exports = class sum extends Process { + geeReducer() { + return 'sum'; + } + async execute(node, context) { - return Commons.reduceInCallback(node, 'sum'); + return node.getData("data"); } }; \ No newline at end of file diff --git a/src/processes/variance.js b/src/processes/variance.js index 421f749..cb18626 100644 --- a/src/processes/variance.js +++ b/src/processes/variance.js @@ -3,8 +3,12 @@ const Commons = require('../processgraph/commons'); module.exports = class variance extends Process { + geeReducer() { + return 'variance'; + } + async execute(node, context) { - return Commons.reduceInCallback(node, 'variance'); + return node.getData("data"); } }; \ No newline at end of file diff --git a/src/processgraph/commons.js b/src/processgraph/commons.js index 25e78d2..89124be 100644 --- a/src/processgraph/commons.js +++ b/src/processgraph/commons.js @@ -3,33 +3,6 @@ const Utils = require('../utils'); module.exports = class ProcessCommons { - static reduceInCallback(node, reducer, dataArg = "data", reducerName = null) { - var isSimpleReducer = node.getProcessGraph().isSimpleReducer(); - var dc = node.getData(dataArg); - if (reducerName === null){ - reducerName = reducer; - } - if (!this.isString(reducerName)){ - throw new Error("The input parameter 'reducerName' is not a string."); - } - var func = data => data.reduce(reducer); - if (isSimpleReducer || dc.isImageCollection()) { - dc.imageCollection(func); - // revert renaming of the bands following to the GEE convention - var bands = dc.getBands(); - var renamedBands = bands.map(bandName => bandName + "_" + reducerName); - var renameBands = image => image.select(renamedBands).rename(bands); - dc.imageCollection(data => data.map(renameBands)); - } - else if (dc.isArray()) { - dc.array(func); - } - else { - throw new Error("Calculating " + reducer + " not supported for given data type."); - } - return dc; - } - static applyInCallback(node, imageProcess, arrayProcess = null, dataArg = "x") { var dc = node.getData(dataArg); if (dc.isImageCollection()) { diff --git a/src/processgraph/processgraph.js b/src/processgraph/processgraph.js index 0aaf912..57018ff 100644 --- a/src/processgraph/processgraph.js +++ b/src/processgraph/processgraph.js @@ -1,6 +1,7 @@ const { ProcessGraph } = require('@openeo/js-commons'); const GeeProcessGraphNode = require('./node'); const Errors = require('../errors'); +const Utils = require('../utils'); module.exports = class GeeProcessGraph extends ProcessGraph { @@ -44,4 +45,9 @@ module.exports = class GeeProcessGraph extends ProcessGraph { this.errors.add(Errors.wrap(error)); } + // ToDo: Remove once we updated to js-commons v0.4.8, it's available there. + getNodeCount() { + return Utils.size(this.nodes); + } + }; \ No newline at end of file From e4e7b3d5a153330e6cbdb60adfca7dbfd3b4f7bb Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Wed, 4 Sep 2019 20:18:15 +0200 Subject: [PATCH 2/6] Second draft for complex process graph reducers. --- docs/s2-evi.json | 201 ++++++++++++++++++++++++++++ package.json | 2 +- src/processes/absolute.js | 2 +- src/processes/arccos.js | 2 +- src/processes/arcsin.js | 2 +- src/processes/arctan.js | 2 +- src/processes/array_element.js | 22 +++ src/processes/array_element.json | 70 ++++++++++ src/processes/ceil.js | 2 +- src/processes/clip.js | 2 +- src/processes/cos.js | 2 +- src/processes/cosh.js | 2 +- src/processes/divide.js | 14 ++ src/processes/divide.json | 93 +++++++++++++ src/processes/exp.js | 2 +- src/processes/first.js | 12 +- src/processes/first.json | 16 ++- src/processes/floor.js | 2 +- src/processes/int.js | 2 +- src/processes/last.js | 12 +- src/processes/last.json | 13 +- src/processes/linear_scale_range.js | 3 +- src/processes/ln.js | 2 +- src/processes/log.js | 2 +- src/processes/max.js | 52 +------ src/processes/mean.js | 2 +- src/processes/median.js | 2 +- src/processes/min.js | 6 +- src/processes/multiply.js | 14 ++ src/processes/multiply.json | 93 +++++++++++++ src/processes/power.js | 3 +- src/processes/product.js | 7 + src/processes/product.json | 44 ++++++ src/processes/reduce.js | 51 +++---- src/processes/reduce.json | 1 - src/processes/round.js | 3 +- src/processes/sd.js | 12 +- src/processes/sin.js | 2 +- src/processes/sinh.js | 2 +- src/processes/sqrt.js | 2 +- src/processes/subtract.js | 14 ++ src/processes/subtract.json | 93 +++++++++++++ src/processes/sum.js | 16 ++- src/processes/tan.js | 2 +- src/processes/tanh.js | 2 +- src/processes/variance.js | 12 +- src/processgraph/commons.js | 136 ++++++++++++------- src/processgraph/context.js | 2 +- src/processgraph/datacube.js | 21 ++- src/processgraph/processgraph.js | 9 -- storage/errors/custom.json | 8 ++ 51 files changed, 891 insertions(+), 204 deletions(-) create mode 100644 docs/s2-evi.json create mode 100644 src/processes/array_element.js create mode 100644 src/processes/array_element.json create mode 100644 src/processes/divide.js create mode 100644 src/processes/divide.json create mode 100644 src/processes/multiply.js create mode 100644 src/processes/multiply.json create mode 100644 src/processes/product.js create mode 100644 src/processes/product.json create mode 100644 src/processes/subtract.js create mode 100644 src/processes/subtract.json diff --git a/docs/s2-evi.json b/docs/s2-evi.json new file mode 100644 index 0000000..521116b --- /dev/null +++ b/docs/s2-evi.json @@ -0,0 +1,201 @@ +{ + "1": { + "process_id": "apply", + "arguments": { + "data": { + "from_node": "mintime" + }, + "process": { + "callback": { + "2": { + "process_id": "linear_scale_range", + "arguments": { + "x": { + "from_argument": "x" + }, + "inputMin": -1, + "inputMax": 1, + "outputMax": 255 + }, + "result": true + } + } + } + }, + "description": "Stretch range from -1 / 1 to 0 / 255 for PNG visualization." + }, + "dc": { + "process_id": "load_collection", + "arguments": { + "id": "COPERNICUS/S2", + "spatial_extent": null, + "temporal_extent": [ + "2018-01-01T00:00:00Z", + "2018-01-31T23:59:59Z" + ], + "bands": [ + "B8", + "B4", + "B2" + ] + }, + "description": "Loading the data; The order of the specified bands is important for the following reduce operation." + }, + "evi": { + "process_id": "reduce", + "arguments": { + "data": { + "from_node": "dc" + }, + "reducer": { + "callback": { + "nir": { + "process_id": "array_element", + "arguments": { + "data": { + "from_argument": "data" + }, + "index": 0 + } + }, + "sub": { + "process_id": "subtract", + "arguments": { + "data": [ + { + "from_node": "nir" + }, + { + "from_node": "red" + } + ] + } + }, + "div": { + "process_id": "divide", + "arguments": { + "data": [ + { + "from_node": "sub" + }, + { + "from_node": "sum" + } + ] + } + }, + "p3": { + "process_id": "product", + "arguments": { + "data": [ + 2.5, + { + "from_node": "div" + } + ] + }, + "result": true + }, + "sum": { + "process_id": "sum", + "arguments": { + "data": [ + 1, + { + "from_node": "nir" + }, + { + "from_node": "p1" + }, + { + "from_node": "p2" + } + ] + } + }, + "red": { + "process_id": "array_element", + "arguments": { + "data": { + "from_argument": "data" + }, + "index": 1 + } + }, + "p1": { + "process_id": "product", + "arguments": { + "data": [ + 6, + { + "from_node": "red" + } + ] + } + }, + "blue": { + "process_id": "array_element", + "arguments": { + "data": { + "from_argument": "data" + }, + "index": 2 + } + }, + "p2": { + "process_id": "product", + "arguments": { + "data": [ + -7.5, + { + "from_node": "blue" + } + ] + } + } + } + }, + "dimension": "bands" + }, + "description": "Compute the EVI. Formula: 2.5 * (NIR - RED) / (1 + NIR + 6*RED + -7.5*BLUE)" + }, + "mintime": { + "process_id": "reduce", + "arguments": { + "data": { + "from_node": "evi" + }, + "reducer": { + "callback": { + "min": { + "process_id": "min", + "arguments": { + "data": { + "from_argument": "data" + } + }, + "result": true + } + } + }, + "dimension": "temporal" + }, + "description": "Compute a minimum time composite by reducing the temporal dimension" + }, + "save": { + "process_id": "save_result", + "arguments": { + "data": { + "from_node": "1" + }, + "format": "PNG", + "options": { + "red": null, + "green": null, + "blue": null, + "gray": null + } + }, + "result": true + } +} \ No newline at end of file diff --git a/package.json b/package.json index 09f2247..45c94c7 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "dependencies": { "@google-cloud/storage": "^3.0.2", "@google/earthengine": "0.1.185", - "@openeo/js-commons": "^0.4.4", + "@openeo/js-commons": "^0.4.7", "ajv": "^6.10.0", "axios": "^0.19.0", "check-disk-space": "^2.1.0", diff --git a/src/processes/absolute.js b/src/processes/absolute.js index 20216d4..d3816d2 100644 --- a/src/processes/absolute.js +++ b/src/processes/absolute.js @@ -4,7 +4,7 @@ const Commons = require('../processgraph/commons'); module.exports = class absolute extends Process { async execute(node, context) { - return Commons.applyInCallback(node, image => image.abs(), array => array.abs()); + return Commons.applyInCallback(node, image => image.abs()); } }; \ No newline at end of file diff --git a/src/processes/arccos.js b/src/processes/arccos.js index d189c56..5173159 100644 --- a/src/processes/arccos.js +++ b/src/processes/arccos.js @@ -4,7 +4,7 @@ const Commons = require('../processgraph/commons'); module.exports = class arccos extends Process { async execute(node, context) { - return Commons.applyInCallback(node, image => image.acos(), array => array.acos()); + return Commons.applyInCallback(node, image => image.acos()); } }; \ No newline at end of file diff --git a/src/processes/arcsin.js b/src/processes/arcsin.js index 3fdf5e8..6a4d903 100644 --- a/src/processes/arcsin.js +++ b/src/processes/arcsin.js @@ -4,7 +4,7 @@ const Commons = require('../processgraph/commons'); module.exports = class arcsin extends Process { async execute(node, context) { - return Commons.applyInCallback(node, image => image.asin(), array => array.asin()); + return Commons.applyInCallback(node, image => image.asin()); } }; \ No newline at end of file diff --git a/src/processes/arctan.js b/src/processes/arctan.js index fd4a7fb..826c908 100644 --- a/src/processes/arctan.js +++ b/src/processes/arctan.js @@ -4,7 +4,7 @@ const Commons = require('../processgraph/commons'); module.exports = class arctan extends Process { async execute(node, context) { - return Commons.applyInCallback(node, image => image.atan(), array => array.atan()); + return Commons.applyInCallback(node, image => image.atan()); } }; \ No newline at end of file diff --git a/src/processes/array_element.js b/src/processes/array_element.js new file mode 100644 index 0000000..cfccaa9 --- /dev/null +++ b/src/processes/array_element.js @@ -0,0 +1,22 @@ +const Process = require('../processgraph/process'); +const Errors = require('../errors'); + +module.exports = class array_element extends Process { + + async execute(node, context) { + var data = node.getArgument("data"); + var index = node.getArgument("index"); + var return_nodata = node.getArgument("return_nodata", false); + + if (Array.isArray(data) && typeof data[index] !== 'undefined') { + return data[index]; + } + else if (return_nodata) { + return null; + } + else { + throw new Errors.IndexOutOfBounds(); + } + } + +}; \ No newline at end of file diff --git a/src/processes/array_element.json b/src/processes/array_element.json new file mode 100644 index 0000000..4de8511 --- /dev/null +++ b/src/processes/array_element.json @@ -0,0 +1,70 @@ +{ + "id": "array_element", + "summary": "Get an element from an array", + "description": "Returns the element at the specified index from the array.", + "categories": [ + "arrays" + ], + "parameter_order": ["data", "index", "return_nodata"], + "parameters": { + "data": { + "description": "An array.", + "schema": { + "type": "array", + "items": { + "description": "Any data type is allowed." + } + }, + "required": true + }, + "index": { + "description": "The zero-based index of the element to retrieve.", + "schema": { + "type": "integer" + }, + "required": true + }, + "return_nodata": { + "description": "By default this process throws an `IndexOutOfBounds` exception if the index is invalid. If you want to return `null` instead, set this flag to `true`.", + "schema": { + "type": "boolean", + "default": false + } + } + }, + "returns": { + "description": "The value of the requested element.", + "schema": { + "description": "Any data type is allowed." + } + }, + "exceptions": { + "IndexOutOfBounds": { + "message": "The array has no element with the specified index." + } + }, + "examples": [ + { + "arguments": { + "data": [9,8,7,6,5], + "index": 2 + }, + "returns": 7 + }, + { + "arguments": { + "data": ["A", "B", "C"], + "index": 0 + }, + "returns": "A" + }, + { + "arguments": { + "data": [], + "index": 0, + "return_nodata": true + }, + "returns": null + } + ] +} \ No newline at end of file diff --git a/src/processes/ceil.js b/src/processes/ceil.js index 92e9a46..eee952e 100644 --- a/src/processes/ceil.js +++ b/src/processes/ceil.js @@ -4,7 +4,7 @@ const Commons = require('../processgraph/commons'); module.exports = class ceil extends Process { async execute(node, context) { - return Commons.applyInCallback(node, image => image.ceil(), array => array.ceil()); + return Commons.applyInCallback(node, image => image.ceil()); } }; \ No newline at end of file diff --git a/src/processes/clip.js b/src/processes/clip.js index cc80d06..773bdeb 100644 --- a/src/processes/clip.js +++ b/src/processes/clip.js @@ -6,7 +6,7 @@ module.exports = class clip extends Process { async execute(node, context) { var min = node.getArgument('min'); var max = node.getArgument('max'); - return Commons.applyInCallback(node, image => image.clamp(min, max)); // Not supported for arrays + return Commons.applyInCallback(node, image => image.clamp(min, max)); } }; \ No newline at end of file diff --git a/src/processes/cos.js b/src/processes/cos.js index c65a998..269067a 100644 --- a/src/processes/cos.js +++ b/src/processes/cos.js @@ -4,7 +4,7 @@ const Commons = require('../processgraph/commons'); module.exports = class cos extends Process { async execute(node, context) { - return Commons.applyInCallback(node, image => image.cos(), array => array.cos()); + return Commons.applyInCallback(node, image => image.cos()); } }; \ No newline at end of file diff --git a/src/processes/cosh.js b/src/processes/cosh.js index bf2a691..bca9e45 100644 --- a/src/processes/cosh.js +++ b/src/processes/cosh.js @@ -4,7 +4,7 @@ const Commons = require('../processgraph/commons'); module.exports = class cosh extends Process { async execute(node, context) { - return Commons.applyInCallback(node, image => image.cosh(), array => array.cosh()); + return Commons.applyInCallback(node, image => image.cosh()); } }; \ No newline at end of file diff --git a/src/processes/divide.js b/src/processes/divide.js new file mode 100644 index 0000000..a61a2e6 --- /dev/null +++ b/src/processes/divide.js @@ -0,0 +1,14 @@ +const Process = require('../processgraph/process'); +const Commons = require('../processgraph/commons'); + +module.exports = class divide extends Process { + + async execute(node, context) { + return Commons.reduceInCallback( + node, + (a,b) => a / b, + (a,b) => a.divide(b) + ); + } + +}; \ No newline at end of file diff --git a/src/processes/divide.json b/src/processes/divide.json new file mode 100644 index 0000000..9ee73b3 --- /dev/null +++ b/src/processes/divide.json @@ -0,0 +1,93 @@ +{ + "id": "divide", + "summary": "Division of a sequence of numbers", + "description": "Divides the first element in a sequential array of numbers by all other elements.\n\nThe computations should follow [IEEE Standard 754](https://ieeexplore.ieee.org/document/4610935) so that for example a division by zero should result in ±infinity if the processing environment supports it. Otherwise an exception must the thrown for incomputable results.\n\nBy default no-data values are ignored. Setting `ignore_nodata` to `false` considers no-data values so that `null` is returned if any element is such a value.", + "categories": [ + "math", + "reducer" + ], + "parameter_order": [ + "data", + "ignore_nodata" + ], + "parameters": { + "data": { + "description": "An array of numbers with at least two elements.", + "schema": { + "type": "array", + "items": { + "type": [ + "number", + "null" + ] + }, + "minItems": 2 + }, + "required": true + }, + "ignore_nodata": { + "description": "Indicates whether no-data values are ignored or not. Ignores them by default. Setting this flag to `false` considers no-data values so that `null` is returned if any value is such a value.", + "schema": { + "type": "boolean", + "default": true + } + } + }, + "returns": { + "description": "The computed result of the sequence of numbers.", + "schema": { + "type": [ + "number", + "null" + ] + } + }, + "exceptions": { + "DivisorMissing": { + "message": "Division requires at least two numbers (a dividend and one or more divisors)." + } + }, + "examples": [ + { + "arguments": { + "data": [ + 15, + 5 + ] + }, + "returns": 3 + }, + { + "arguments": { + "data": [ + -2, + 4, + 2.5 + ] + }, + "returns": -0.2 + }, + { + "arguments": { + "data": [ + 1, + null + ], + "ignore_nodata": false + }, + "returns": null + } + ], + "links": [ + { + "rel": "about", + "href": "http://mathworld.wolfram.com/Division.html", + "title": "Division explained by Wolfram MathWorld" + }, + { + "rel": "about", + "href": "https://ieeexplore.ieee.org/document/4610935", + "title": "IEEE Standard 754-2008 for Floating-Point Arithmetic" + } + ] +} \ No newline at end of file diff --git a/src/processes/exp.js b/src/processes/exp.js index 9f53083..253f2e6 100644 --- a/src/processes/exp.js +++ b/src/processes/exp.js @@ -4,7 +4,7 @@ const Commons = require('../processgraph/commons'); module.exports = class exp extends Process { async execute(node, context) { - return Commons.applyInCallback(node, image => image.exp(), array => array.exp()); + return Commons.applyInCallback(node, image => image.exp()); // TODO: implement numbers //var p = node.getArgument("p"); //return ee.Number(p).exp().getInfo(); diff --git a/src/processes/first.js b/src/processes/first.js index f469d39..e3febf8 100644 --- a/src/processes/first.js +++ b/src/processes/first.js @@ -3,12 +3,12 @@ const Commons = require('../processgraph/commons'); module.exports = class first extends Process { - geeReducer() { - return 'first'; - } + geeReducer(node) { + return node.getArgument('ignore_nodata', true) ? 'firstNonNull' : 'first'; + } - async execute(node, context) { - return node.getData("data"); - } + async execute(node, context) { + throw "Not implemented yet."; + } }; \ No newline at end of file diff --git a/src/processes/first.json b/src/processes/first.json index 03b52df..911c796 100644 --- a/src/processes/first.json +++ b/src/processes/first.json @@ -7,9 +7,9 @@ "reducer" ], "parameter_order": [ - "data" + "data", + "ignore_nodata" ], - "gee:custom": true, "parameters": { "data": { "description": "An array with elements of any data type. An empty array resolves always with `null`.", @@ -20,6 +20,13 @@ } }, "required": true + }, + "ignore_nodata": { + "description": "Indicates whether no-data values are ignored or not. Ignores them by default. Setting this flag to `false` considers no-data values so that `null` is returned if the first value is such a value.", + "schema": { + "type": "boolean", + "default": true + } } }, "returns": { @@ -56,9 +63,10 @@ null, 2, 3 - ] + ], + "ignore_nodata": false }, - "returns": 2 + "returns": null }, { "description": "The input array is empty: return `null`.", diff --git a/src/processes/floor.js b/src/processes/floor.js index af05c4f..2f1045f 100644 --- a/src/processes/floor.js +++ b/src/processes/floor.js @@ -4,7 +4,7 @@ const Commons = require('../processgraph/commons'); module.exports = class floor extends Process { async execute(node, context) { - return Commons.applyInCallback(node, image => image.floor(), array => array.floor()); + return Commons.applyInCallback(node, image => image.floor()); } }; \ No newline at end of file diff --git a/src/processes/int.js b/src/processes/int.js index f088b09..faa29e0 100644 --- a/src/processes/int.js +++ b/src/processes/int.js @@ -4,7 +4,7 @@ const Commons = require('../processgraph/commons'); module.exports = class int extends Process { async execute(node, context) { - return Commons.applyInCallback(node, image => image.int(), array => array.int()); + return Commons.applyInCallback(node, image => image.int()); } }; \ No newline at end of file diff --git a/src/processes/last.js b/src/processes/last.js index e341aff..31d2e8d 100644 --- a/src/processes/last.js +++ b/src/processes/last.js @@ -3,12 +3,12 @@ const Commons = require('../processgraph/commons'); module.exports = class last extends Process { - geeReducer() { - return 'last'; - } + geeReducer(node) { + return node.getArgument('ignore_nodata', true) ? 'lastNonNull' : 'last'; + } - async execute(node, context) { - return node.getData("data"); - } + async execute(node, context) { + throw "Not implemented yet."; + } }; \ No newline at end of file diff --git a/src/processes/last.json b/src/processes/last.json index 6ee11a6..d470525 100644 --- a/src/processes/last.json +++ b/src/processes/last.json @@ -10,7 +10,6 @@ "data", "ignore_nodata" ], - "gee:custom": true, "parameters": { "data": { "description": "An array with elements of any data type. An empty array resolves always with `null`.", @@ -21,6 +20,13 @@ } }, "required": true + }, + "ignore_nodata": { + "description": "Indicates whether no-data values are ignored or not. Ignores them by default. Setting this flag to `false` considers no-data values so that `null` is returned if the last value is such a value.", + "schema": { + "type": "boolean", + "default": true + } } }, "returns": { @@ -57,9 +63,10 @@ 0, 1, null - ] + ], + "ignore_nodata": false }, - "returns": 1 + "returns": null }, { "description": "The input array is empty: return `null`.", diff --git a/src/processes/linear_scale_range.js b/src/processes/linear_scale_range.js index ccb78e7..8697c44 100644 --- a/src/processes/linear_scale_range.js +++ b/src/processes/linear_scale_range.js @@ -16,8 +16,7 @@ module.exports = class linear_scale_range extends Process { var inputMax = node.getArgument('inputMax'); var outputMin = node.getArgument('outputMin', 0); var outputMax = node.getArgument('outputMax', 1); - var process = data => this.process(data, inputMin, inputMax, outputMin, outputMax); - return Commons.applyInCallback(node, process, process); + return Commons.applyInCallback(node, data => this.process(data, inputMin, inputMax, outputMin, outputMax)); } }; \ No newline at end of file diff --git a/src/processes/ln.js b/src/processes/ln.js index b860abd..60c4f9e 100644 --- a/src/processes/ln.js +++ b/src/processes/ln.js @@ -4,7 +4,7 @@ const Commons = require('../processgraph/commons'); module.exports = class ln extends Process { async execute(node, context) { - return Commons.applyInCallback(node, image => image.log(), array => array.log()); + return Commons.applyInCallback(node, image => image.log()); } }; \ No newline at end of file diff --git a/src/processes/log.js b/src/processes/log.js index 4a1d112..58864a4 100644 --- a/src/processes/log.js +++ b/src/processes/log.js @@ -5,7 +5,7 @@ module.exports = class log extends Process { async execute(node, context) { // GEE only supports log with base 10 (or ln). - return Commons.applyInCallback(node, image => image.log10(), array => array.log10()); + return Commons.applyInCallback(node, image => image.log10()); } }; \ No newline at end of file diff --git a/src/processes/max.js b/src/processes/max.js index 87abb79..cd361de 100644 --- a/src/processes/max.js +++ b/src/processes/max.js @@ -8,53 +8,11 @@ module.exports = class max extends Process { } async execute(node, context) { - var list = node.getArgument('data'); - if (list.length <= 1) { - // ToDo: Invalid, handle appropriately - return; - } - - for(var i = 1; i < list.length; i++) { - var first = list[i-1]; - var second = list[i]; - if (typeof first === 'number' && typeof second === 'number') { - list[i] = Math.max(first, second); - } - else if (first.isImageCollection()) { - if (typeof second === 'number') { - list[i] = first.imageCollection(ic => ic.map(img => img.max(second))); - } - else if (second.isImage()) { - list[i] = second.image(img => img.max(first)); - } - else { - throw "Not supported"; - } - } - else if (first.isImage()) { - if (typeof second === 'number' || second.isImage()) { - list[i] = first.image(img => img.max(second)); - } - else if (second.isImageCollection()) { - list[i] = second.imageCollection(ic => ic.map(img => img.max(first))); - } - else { - throw "Not supported"; - } - } - else if (first.isArray()) { - if (typeof second === 'number' || second.isArray()) { - list[i] = first.array(arr => arr.max(second)); // Does this work for numbers? - } - else { - throw "Not supported"; - } - } - else { - throw "Not supported"; - } - } - return list[list.length-1]; + return Commons.reduceInCallback( + node, + (a,b) => Math.max(a,b), + (a,b) => a.max(b) + ); } }; \ No newline at end of file diff --git a/src/processes/mean.js b/src/processes/mean.js index c9ddb3e..e671acf 100644 --- a/src/processes/mean.js +++ b/src/processes/mean.js @@ -8,7 +8,7 @@ module.exports = class mean extends Process { } async execute(node, context) { - return node.getData("data"); + throw "Not implemented yet."; } }; \ No newline at end of file diff --git a/src/processes/median.js b/src/processes/median.js index 2e464bd..386f096 100644 --- a/src/processes/median.js +++ b/src/processes/median.js @@ -8,7 +8,7 @@ module.exports = class median extends Process { } async execute(node, context) { - return node.getData("data"); + throw "Not implemented yet."; } }; \ No newline at end of file diff --git a/src/processes/min.js b/src/processes/min.js index 9342132..617fbc8 100644 --- a/src/processes/min.js +++ b/src/processes/min.js @@ -8,7 +8,11 @@ module.exports = class min extends Process { } async execute(node, context) { - return node.getData("data"); + return Commons.reduceInCallback( + node, + (a,b) => Math.min(a,b), + (a,b) => a.min(b) + ); } }; \ No newline at end of file diff --git a/src/processes/multiply.js b/src/processes/multiply.js new file mode 100644 index 0000000..5d0d502 --- /dev/null +++ b/src/processes/multiply.js @@ -0,0 +1,14 @@ +const Process = require('../processgraph/process'); +const Commons = require('../processgraph/commons'); + +module.exports = class multiply extends Process { + + async execute(node, context) { + return Commons.reduceInCallback( + node, + (a,b) => a * b, + (a,b) => a.multiply(b) + ); + } + +}; \ No newline at end of file diff --git a/src/processes/multiply.json b/src/processes/multiply.json new file mode 100644 index 0000000..4615a93 --- /dev/null +++ b/src/processes/multiply.json @@ -0,0 +1,93 @@ +{ + "id": "multiply", + "summary": "Multiplication of a sequence of numbers", + "description": "Multiplies all elements in a sequential array of numbers and returns the computed product.\n\nThe computations should follow [IEEE Standard 754](https://ieeexplore.ieee.org/document/4610935) whenever the processing environment supports it. Otherwise an exception must the thrown for incomputable results.\n\nBy default no-data values are ignored. Setting `ignore_nodata` to `false` considers no-data values so that `null` is returned if any element is such a value.", + "categories": [ + "math", + "reducer" + ], + "parameter_order": [ + "data", + "ignore_nodata" + ], + "parameters": { + "data": { + "description": "An array of numbers with at least two elements.", + "schema": { + "type": "array", + "items": { + "type": [ + "number", + "null" + ] + }, + "minItems": 2 + }, + "required": true + }, + "ignore_nodata": { + "description": "Indicates whether no-data values are ignored or not. Ignores them by default. Setting this flag to `false` considers no-data values so that `null` is returned if any value is such a value.", + "schema": { + "type": "boolean", + "default": true + } + } + }, + "returns": { + "description": "The computed product of the sequence of numbers.", + "schema": { + "type": [ + "number", + "null" + ] + } + }, + "exceptions": { + "MultiplicandMissing": { + "message": "Multiplication requires at least two numbers." + } + }, + "examples": [ + { + "arguments": { + "data": [ + 5, + 0 + ] + }, + "returns": 0 + }, + { + "arguments": { + "data": [ + -2, + 4, + 2.5 + ] + }, + "returns": -20 + }, + { + "arguments": { + "data": [ + 1, + null + ], + "ignore_nodata": false + }, + "returns": null + } + ], + "links": [ + { + "rel": "about", + "href": "http://mathworld.wolfram.com/Product.html", + "title": "Product explained by Wolfram MathWorld" + }, + { + "rel": "about", + "href": "https://ieeexplore.ieee.org/document/4610935", + "title": "IEEE Standard 754-2008 for Floating-Point Arithmetic" + } + ] +} \ No newline at end of file diff --git a/src/processes/power.js b/src/processes/power.js index 0cff211..ce7ac9e 100644 --- a/src/processes/power.js +++ b/src/processes/power.js @@ -5,8 +5,7 @@ module.exports = class power extends Process { async execute(node, context) { var power = node.getArgument('p'); - return Commons.applyInCallback(node, image => image.pow(power), - array => array.pow(power), "base"); + return Commons.applyInCallback(node, image => image.pow(power), "base"); } }; \ No newline at end of file diff --git a/src/processes/product.js b/src/processes/product.js new file mode 100644 index 0000000..51d3878 --- /dev/null +++ b/src/processes/product.js @@ -0,0 +1,7 @@ +const multiply = require('./multiply'); + +module.exports = class product extends multiply { + + // product is an alias for multiply + +}; \ No newline at end of file diff --git a/src/processes/product.json b/src/processes/product.json new file mode 100644 index 0000000..e27a1eb --- /dev/null +++ b/src/processes/product.json @@ -0,0 +1,44 @@ +{ + "id": "product", + "summary": "Multiplication of a sequence of numbers", + "description": "This process is an exact alias for the `multiply` process. See ``multiply()`` for more information.", + "categories": [ + "math", + "reducer" + ], + "parameter_order": [ + "data", + "ignore_nodata" + ], + "parameters": { + "data": { + "description": "See ``multiply()`` for more information.", + "schema": { + "type": "array", + "items": { + "type": [ + "number", + "null" + ] + } + }, + "required": true + }, + "ignore_nodata": { + "description": "See ``multiply()`` for more information.", + "schema": { + "type": "boolean", + "default": true + } + } + }, + "returns": { + "description": "See ``multiply()`` for more information.", + "schema": { + "type": [ + "number", + "null" + ] + } + } +} \ No newline at end of file diff --git a/src/processes/reduce.js b/src/processes/reduce.js index b9fac52..c0e4972 100644 --- a/src/processes/reduce.js +++ b/src/processes/reduce.js @@ -1,5 +1,6 @@ const Process = require('../processgraph/process'); const Errors = require('../errors'); +const DataCube = require('../processgraph/datacube'); // TODO: do we have to change this/reduce the dimension if we get multiple arguments back, e.g. from quantiles? module.exports = class reduce extends Process { @@ -29,26 +30,23 @@ module.exports = class reduce extends Process { reason: 'The specified reducer is invalid.' }); } - resultDataCube = this.reduceSimple(dc, process.geeReducer()); + resultDataCube = this.reduceSimple(dc, process.geeReducer(node)); } else { + // This is a complex reducer var values; + var ic = dc.imageCollection(); if (dimension.type === 'temporal') { - var ic = data.imageCollection(); - values = ic.toList(); + values = ic.toList(ic.size()); } else if (dimension.type === 'bands') { - var ic = data.imageCollection(); - // ToDo: Ensure that the bands have a fixed order! - values = data.getBands().map(band => ic.select(band)); + values = dimension.getValues().map(band => ic.select(band)); } - // This is a complex reducer -// var resultNode = this.reduceComplex(); var resultNode = await callback.execute({ - data: ic, - values: values + data: values }); - resultDataCube = resultNode.getResult(); + resultDataCube = new DataCube(dc); + resultDataCube.setData(resultNode.getResult()); } resultDataCube.dropDimension(dimensionName); return resultDataCube; @@ -64,29 +62,22 @@ module.exports = class reduce extends Process { } } - var bands = dc.getBands(); - var renamedBands = bands.map(bandName => bandName + "_" + reducerName); - if (dc.isImageCollection()) { - dc.imageCollection(data => data.reduce(reducerFunc).map( - // revert renaming of the bands following to the GEE convention - image => image.select(renamedBands).rename(bands) - )); - } - else if (dc.isImage()) { - // reduce and revert renaming of the bands following to the GEE convention - dc.image(img => img.reduce(reducerFunc).select(renamedBands).rename(bands)); - } - else if (dc.isArray()) { - dc.array(data => data.reduce(reducerFunc)); - } - else { + if (!dc.isImageCollection() && !dc.isImage()) { throw new Error("Calculating " + reducerName + " not supported for given data type."); } - return dc; - } + + dc.imageCollection(data => data.reduce(reducerFunc)); - reduceComplex() { + // revert renaming of the bands following to the GEE convention + var bands = dc.getBands(); + if (Array.isArray(bands) && bands.length > 0) { + var renamedBands = bands.map(bandName => bandName + "_" + reducerName); + dc.imageCollection(data => data.map( + img => img.select(renamedBands).rename(bands) + )); + } + return dc; } }; \ No newline at end of file diff --git a/src/processes/reduce.json b/src/processes/reduce.json index 633acc7..83719a2 100644 --- a/src/processes/reduce.json +++ b/src/processes/reduce.json @@ -6,7 +6,6 @@ "cubes", "reducer" ], - "gee:reducer": true, "gee:custom": true, "parameter_order": ["data", "reducer", "dimension"], "parameters": { diff --git a/src/processes/round.js b/src/processes/round.js index aa62fee..754cf7d 100644 --- a/src/processes/round.js +++ b/src/processes/round.js @@ -16,8 +16,7 @@ module.exports = class round extends Process { async execute(node, context) { var p = node.getArgument("p"); - var process = data => this.process(data, p); - return Commons.applyInCallback(node, process, process); + return Commons.applyInCallback(node, data => this.process(data, p)); } }; \ No newline at end of file diff --git a/src/processes/sd.js b/src/processes/sd.js index 7a1882e..0aa03e5 100644 --- a/src/processes/sd.js +++ b/src/processes/sd.js @@ -3,12 +3,12 @@ const Commons = require('../processgraph/commons'); module.exports = class sd extends Process { - geeReducer() { - return 'stdDev'; - } + geeReducer() { + return 'stdDev'; + } - async execute(node, context) { - return node.getData("data"); - } + async execute(node, context) { + throw "Not implemented yet."; + } }; \ No newline at end of file diff --git a/src/processes/sin.js b/src/processes/sin.js index 63f18f6..c159779 100644 --- a/src/processes/sin.js +++ b/src/processes/sin.js @@ -4,7 +4,7 @@ const Commons = require('../processgraph/commons'); module.exports = class sin extends Process { async execute(node, context) { - return Commons.applyInCallback(node, image => image.sin(), array => array.sin()); + return Commons.applyInCallback(node, image => image.sin()); } }; \ No newline at end of file diff --git a/src/processes/sinh.js b/src/processes/sinh.js index 3fbc6cf..8bb0e89 100644 --- a/src/processes/sinh.js +++ b/src/processes/sinh.js @@ -4,7 +4,7 @@ const Commons = require('../processgraph/commons'); module.exports = class sinh extends Process { async execute(node, context) { - return Commons.applyInCallback(node, image => image.sinh(), array => array.sinh()); + return Commons.applyInCallback(node, image => image.sinh()); } }; \ No newline at end of file diff --git a/src/processes/sqrt.js b/src/processes/sqrt.js index 2431a4d..9ff4940 100644 --- a/src/processes/sqrt.js +++ b/src/processes/sqrt.js @@ -4,7 +4,7 @@ const Commons = require('../processgraph/commons'); module.exports = class sqrt extends Process { async execute(node, context) { - return Commons.applyInCallback(node, image => image.sqrt(), array => array.sqrt()); + return Commons.applyInCallback(node, image => image.sqrt()); } }; \ No newline at end of file diff --git a/src/processes/subtract.js b/src/processes/subtract.js new file mode 100644 index 0000000..502dafb --- /dev/null +++ b/src/processes/subtract.js @@ -0,0 +1,14 @@ +const Process = require('../processgraph/process'); +const Commons = require('../processgraph/commons'); + +module.exports = class subtract extends Process { + + async execute(node, context) { + return Commons.reduceInCallback( + node, + (a,b) => a - b, + (a,b) => a.subtract(b) + ); + } + +}; \ No newline at end of file diff --git a/src/processes/subtract.json b/src/processes/subtract.json new file mode 100644 index 0000000..c83db42 --- /dev/null +++ b/src/processes/subtract.json @@ -0,0 +1,93 @@ +{ + "id": "subtract", + "summary": "Subtraction of a sequence of numbers", + "description": "Takes the first element of a sequential array of numbers and subtracts all other elements from it.\n\nThe computations should follow [IEEE Standard 754](https://ieeexplore.ieee.org/document/4610935) whenever the processing environment supports it. Otherwise an exception must the thrown for incomputable results.\n\nBy default no-data values are ignored. Setting `ignore_nodata` to `false` considers no-data values so that `null` is returned if any element is such a value.", + "categories": [ + "math", + "reducer" + ], + "parameter_order": [ + "data", + "ignore_nodata" + ], + "parameters": { + "data": { + "description": "An array of numbers with at least two elements.", + "schema": { + "type": "array", + "items": { + "type": [ + "number", + "null" + ] + }, + "minItems": 2 + }, + "required": true + }, + "ignore_nodata": { + "description": "Indicates whether no-data values are ignored or not. Ignores them by default. Setting this flag to `false` considers no-data values so that `null` is returned if any value is such a value.", + "schema": { + "type": "boolean", + "default": true + } + } + }, + "returns": { + "description": "The computed result of the sequence of numbers.", + "schema": { + "type": [ + "number", + "null" + ] + } + }, + "exceptions": { + "SubtrahendMissing": { + "message": "Subtraction requires at least two numbers (a minuend and one or more subtrahends)." + } + }, + "examples": [ + { + "arguments": { + "data": [ + 5, + 10 + ] + }, + "returns": -5 + }, + { + "arguments": { + "data": [ + -2, + 4, + -2 + ] + }, + "returns": -4 + }, + { + "arguments": { + "data": [ + 1, + null + ], + "ignore_nodata": false + }, + "returns": null + } + ], + "links": [ + { + "rel": "about", + "href": "http://mathworld.wolfram.com/Subtraction.html", + "title": "Subtraction explained by Wolfram MathWorld" + }, + { + "rel": "about", + "href": "https://ieeexplore.ieee.org/document/4610935", + "title": "IEEE Standard 754-2008 for Floating-Point Arithmetic" + } + ] +} \ No newline at end of file diff --git a/src/processes/sum.js b/src/processes/sum.js index ef7c2e5..bbd27eb 100644 --- a/src/processes/sum.js +++ b/src/processes/sum.js @@ -3,12 +3,16 @@ const Commons = require('../processgraph/commons'); module.exports = class sum extends Process { - geeReducer() { - return 'sum'; - } + geeReducer() { + return 'sum'; + } - async execute(node, context) { - return node.getData("data"); - } + async execute(node, context) { + return Commons.reduceInCallback( + node, + (a,b) => a + b, + (a,b) => a.add(b) + ); + } }; \ No newline at end of file diff --git a/src/processes/tan.js b/src/processes/tan.js index 3433cc6..932a816 100644 --- a/src/processes/tan.js +++ b/src/processes/tan.js @@ -4,7 +4,7 @@ const Commons = require('../processgraph/commons'); module.exports = class tan extends Process { async execute(node, context) { - return Commons.applyInCallback(node, image => image.tan(), array => array.tan()); + return Commons.applyInCallback(node, image => image.tan()); } }; \ No newline at end of file diff --git a/src/processes/tanh.js b/src/processes/tanh.js index deec7d7..724df49 100644 --- a/src/processes/tanh.js +++ b/src/processes/tanh.js @@ -4,7 +4,7 @@ const Commons = require('../processgraph/commons'); module.exports = class tanh extends Process { async execute(node, context) { - return Commons.applyInCallback(node, image => image.tanh(), array => array.tanh()); + return Commons.applyInCallback(node, image => image.tanh()); } }; \ No newline at end of file diff --git a/src/processes/variance.js b/src/processes/variance.js index cb18626..9ec9e0d 100644 --- a/src/processes/variance.js +++ b/src/processes/variance.js @@ -3,12 +3,12 @@ const Commons = require('../processgraph/commons'); module.exports = class variance extends Process { - geeReducer() { - return 'variance'; - } + geeReducer() { + return 'variance'; + } - async execute(node, context) { - return node.getData("data"); - } + async execute(node, context) { + throw "Not implemented yet."; + } }; \ No newline at end of file diff --git a/src/processgraph/commons.js b/src/processgraph/commons.js index 89124be..e3472b4 100644 --- a/src/processgraph/commons.js +++ b/src/processgraph/commons.js @@ -1,9 +1,97 @@ const Errors = require('../errors'); const Utils = require('../utils'); +const DataCube = require('./datacube'); module.exports = class ProcessCommons { - static applyInCallback(node, imageProcess, arrayProcess = null, dataArg = "x") { + static reduceInCallback(node, jsReducer, imgReducer, dataArg = "data") { + var list = node.getArgument(dataArg); + if (!Array.isArray(list) || list.length <= 1) { + throw new Errors.ProcessArgumentInvalid({ + process: node.process_id, + argument: dataArg, + reason: "Not enough elements." + }); + } + + var result; + for(var i = 1; i < list.length; i++) { + var valA = list[i-1]; + var valB = list[i]; + var dataCubeA = new DataCube(null, valA); + var dataCubeB = new DataCube(null, valB); + if (typeof valA === 'number') { + var imgA = ee.Image(valA); + if (typeof valB === 'number') { + result = jsReducer(valA, valB); + } + else if (dataCubeB.isImage()) { + result = imgReducer(imgA, dataCubeB.image()); + } + else if (dataCubeB.isImageCollection()) { + result = dataCubeB.imageCollection(ic => ic.map(imgB => imgReducer(imgA, imgB))); + } + else { + throw new Errors.ProcessArgumentInvalid({ + process: node.process_id, + argument: dataArg, + reason: "Reducing number with unknown type not supported (index: "+i+")" + }); + } + } + else if (dataCubeA.isImageCollection()) { + var collA = dataCubeA.imageCollection(); + if (typeof valB === 'number' || dataCubeB.isImage()) { + var imgB = typeof valB === 'number' ? ee.Image(valB) : dataCubeB.image(); + result = collA.map(imgA => imgReducer(imgA, imgB)); + } + else if (dataCubeB.isImageCollection()) { + var collB = dataCubeB.imageCollection(); + var listA = collA.toList(collA.size()); + var listB = collB.toList(collB.size()); + result = collA.map(imgA => { + var index = listA.indexOf(imgA); + var imgB = listB.get(index); + return imgReducer(imgA, imgB); + }); + } + else { + throw new Errors.ProcessArgumentInvalid({ + process: node.process_id, + argument: dataArg, + reason: "Reducing image collection with unknown type not supported (index: "+i+")" + }); + } + } + else if (dataCubeA.isImage()) { + var imgA = dataCubeA.image(); + if (typeof valB === 'number' || dataCubeB.isImage()) { + var imgB = typeof valB === 'number' ? ee.Image(valB) : dataCubeB.image(); + result = imgReducer(imgA, imgB); + } + else if (dataCubeB.isImageCollection()) { + result = dataCubeB.imageCollection(ic => ic.map(imgB => imgReducer(imgA, imgB))); + } + else { + throw new Errors.ProcessArgumentInvalid({ + process: node.process_id, + argument: dataArg, + reason: "Reducing image with unknown type not supported (index: "+i+")" + }); + } + } + else { + throw new Errors.ProcessArgumentInvalid({ + process: node.process_id, + argument: dataArg, + reason: "Reducing an unknwon type is not supported (index: "+i+")" + }); + } + } + return result; + } + + static applyInCallback(node, imageProcess, dataArg = "x") { var dc = node.getData(dataArg); if (dc.isImageCollection()) { dc.imageCollection(data => data.map(imageProcess)); @@ -11,11 +99,8 @@ module.exports = class ProcessCommons { else if (dc.isImage()){ dc.image(imageProcess); } - else if (dc.isArray() && arrayProcess) { - dc.array(arrayProcess); - } else { - throw "Calculating " + process + " not supported for given data type."; + throw "Calculating " + node.process_id + " not supported for given data type."; } return dc; } @@ -66,45 +151,4 @@ module.exports = class ProcessCommons { return dc; } - //TODO - /* - static filter(dc, expression, dimensionName){ - var dimension = dc.findSingleDimension(dimensionName); - var values = dimension.getValues(); - //var selection = => ; - //var values_filtered = await expression.execute({x: data}); - dc.findSingleDimension(dimensionName).setValues(values_filtered); - - return dc; - }*/ - - static dimOEO2dimGEE(bandName, parName=null){ - var dimensionString = bandName; - if (parName !== null){ - dimensionString += "_" + parName - } - - return dimensionString - } - - static dimGEE2dimOEO(dimensionString){ - var stringParts = dimensionString.split('_'); - var bandName = (stringParts.length > 0) ? stringParts[0] : null; - var parName = (stringParts.length > 1) ? stringParts[1] : null; - - return [bandName, parName] - } - - static isString(x) { - return typeof(x) === 'string' || x instanceof String; - } - - static isNumber(x) { - return typeof(x) === 'string' || x instanceof Number; - } - - static isBoolean(x) { - return typeof(x) === 'boolean' || x instanceof Boolean; - } - }; diff --git a/src/processgraph/context.js b/src/processgraph/context.js index aafea60..4b56533 100644 --- a/src/processgraph/context.js +++ b/src/processgraph/context.js @@ -66,7 +66,7 @@ module.exports = class ProcessingContext { } return new Promise((resolve, reject) => { var visBands = null; - var availableBands = dataCube.getBands(); + var availableBands = dataCube.getBands(true); var parameters = dataCube.getOutputFormatParameters(); // this will be important/used in the future if (parameters.red && parameters.green && parameters.blue){ visBands = [parameters.red, parameters.green, parameters.blue]; diff --git a/src/processgraph/datacube.js b/src/processgraph/datacube.js index 9f1b953..2a80b05 100644 --- a/src/processgraph/datacube.js +++ b/src/processgraph/datacube.js @@ -4,8 +4,8 @@ const proj4 = require('proj4'); module.exports = class DataCube { - constructor(sourceDataCube = null) { - this.data = null; + constructor(sourceDataCube = null, data = undefined) { + this.data = data; this.dimensions = {}; this.output = { format: null, @@ -13,7 +13,9 @@ module.exports = class DataCube { }; if (sourceDataCube instanceof DataCube) { - this.data = sourceDataCube.data; + if (data === undefined) { + this.data = sourceDataCube.data; + } this.output = Object.assign({}, sourceDataCube.output); for(var i in sourceDataCube.dimensions) { this.dimensions[i] = new Dimension(this, sourceDataCube.dimensions[i]); @@ -225,8 +227,17 @@ module.exports = class DataCube { return this.dimT(); } - getBands() { - return this.dimBands().getValues(); + getBands(force = false) { + try { + return this.dimBands().getValues(); + } catch(e) { + if (force) { + return this.image().bandNames().getInfo(); + } + else { + return []; + } + } } setBands(bands) { diff --git a/src/processgraph/processgraph.js b/src/processgraph/processgraph.js index 57018ff..5787b41 100644 --- a/src/processgraph/processgraph.js +++ b/src/processgraph/processgraph.js @@ -32,15 +32,6 @@ module.exports = class GeeProcessGraph extends ProcessGraph { return await process.execute(node, this.context); } - isSimpleReducer() { - return (this.isReducer() && this.nodes.length === 1); - } - - isReducer() { - var process = this.getParentProcess(); - return (process && process.schema['gee:reducer'] === true); - } - addError(error) { this.errors.add(Errors.wrap(error)); } diff --git a/storage/errors/custom.json b/storage/errors/custom.json index fcafa0b..e507833 100644 --- a/storage/errors/custom.json +++ b/storage/errors/custom.json @@ -31,5 +31,13 @@ "tags": [ "Processes" ] + }, + "IndexOutOfBounds": { + "description": "The array has no element with the specified index.", + "message": "The array has no element with the specified index.", + "http": 400, + "tags": [ + "Processes" + ] } } \ No newline at end of file From e646adb3c88c279db578c34b915ceebc35b6621a Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 5 Sep 2019 21:16:14 +0200 Subject: [PATCH 3/6] Speed improvements by avoiding getInfo() --- src/api/services.js | 8 ++------ src/models/servicestore.js | 9 ++++++++- src/processes/exp.js | 3 --- src/processes/reduce.js | 1 + src/processgraph/commons.js | 2 +- src/processgraph/context.js | 18 +++++++----------- src/processgraph/datacube.js | 28 ++++++++-------------------- src/processgraph/processgraph.js | 7 +------ src/utils.js | 13 +++++++++++++ 9 files changed, 41 insertions(+), 48 deletions(-) diff --git a/src/api/services.js b/src/api/services.js index 51a9519..36268c9 100644 --- a/src/api/services.js +++ b/src/api/services.js @@ -21,10 +21,6 @@ module.exports = class ServicesAPI { return Promise.resolve(); } - calculateXYZRect(x, y, z) { - return ee.Geometry.Rectangle(this.storage.calculateXYZRect(x, y, z), 'EPSG:4326'); - } - getXYZ(req, res, next) { var query = { // Tiles are always public! @@ -40,12 +36,12 @@ module.exports = class ServicesAPI { } try { - var rect = this.calculateXYZRect(req.params.x, req.params.y, req.params.z); + var rect = this.storage.calculateXYZRect(req.params.x, req.params.y, req.params.z); var context = this.context.processingContext(req); // Update user id to the user id, which stored the job. See https://github.com/Open-EO/openeo-earthengine-driver/issues/19 context.setUserId(service.user_id); var pg = new ProcessGraph(service.process_graph, context); - pg.optimizeLoadCollectionRect(this.storage.calculateXYZRect(req.params.x, req.params.y, req.params.z)); + pg.optimizeLoadCollectionRect(rect); pg.execute() .then(resultNode => context.retrieveResults(resultNode.getResult(), '256x256', rect)) .then(url => { diff --git a/src/models/servicestore.js b/src/models/servicestore.js index ba87edc..8312ca7 100644 --- a/src/models/servicestore.js +++ b/src/models/servicestore.js @@ -39,7 +39,14 @@ module.exports = class ServiceStore { var xMax = Math.max(nw_lng, se_lng); var yMin = Math.min(nw_lat, se_lat); var yMax = Math.max(nw_lat, se_lat); - return [xMin, yMin, xMax, yMax]; + + return { + west: xMin, + east: xMax, + south: yMin, + north: yMax, +// crs: 'EPSG:3857' + }; } }; \ No newline at end of file diff --git a/src/processes/exp.js b/src/processes/exp.js index 253f2e6..e1fad9d 100644 --- a/src/processes/exp.js +++ b/src/processes/exp.js @@ -5,9 +5,6 @@ module.exports = class exp extends Process { async execute(node, context) { return Commons.applyInCallback(node, image => image.exp()); - // TODO: implement numbers - //var p = node.getArgument("p"); - //return ee.Number(p).exp().getInfo(); } }; \ No newline at end of file diff --git a/src/processes/reduce.js b/src/processes/reduce.js index c0e4972..a287340 100644 --- a/src/processes/reduce.js +++ b/src/processes/reduce.js @@ -48,6 +48,7 @@ module.exports = class reduce extends Process { resultDataCube = new DataCube(dc); resultDataCube.setData(resultNode.getResult()); } + // ToDo: We don't know at this point how the bands in the GEE images/imagecollections are called. resultDataCube.dropDimension(dimensionName); return resultDataCube; } diff --git a/src/processgraph/commons.js b/src/processgraph/commons.js index e3472b4..40ad1df 100644 --- a/src/processgraph/commons.js +++ b/src/processgraph/commons.js @@ -108,7 +108,7 @@ module.exports = class ProcessCommons { static filterBbox(dc, bbox, process_id, paramName) { try { dc.setSpatialExtent(bbox); - var geom = dc.getSpatialExtentAsGeeGeometry(); + var geom = ee.Geometry.Rectangle([bbox.west, bbox.south, bbox.east, bbox.north], bbox.crs || 'EPSG:4326'); dc.imageCollection(ic => ic.filterBounds(geom)); return dc; } catch (e) { diff --git a/src/processgraph/context.js b/src/processgraph/context.js index 4b56533..3ce9b98 100644 --- a/src/processgraph/context.js +++ b/src/processgraph/context.js @@ -56,18 +56,17 @@ module.exports = class ProcessingContext { } // TODO: the selection of formats and bands is really strict at the moment, maybe some of them are too strict - async retrieveResults(dataCube, size = 2000, bounds = null) { + async retrieveResults(dataCube, size = 2000, bbox = null) { var format = dataCube.getOutputFormat() || "jpeg"; switch(format.toLowerCase()) { case 'jpeg': case 'png': - if (!bounds) { - bounds = dataCube.getSpatialExtentAsGeeGeometry(); + if (!bbox) { + bbox = dataCube.getSpatialExtent(); } return new Promise((resolve, reject) => { var visBands = null; - var availableBands = dataCube.getBands(true); - var parameters = dataCube.getOutputFormatParameters(); // this will be important/used in the future + var parameters = dataCube.getOutputFormatParameters(); if (parameters.red && parameters.green && parameters.blue){ visBands = [parameters.red, parameters.green, parameters.blue]; } @@ -81,15 +80,12 @@ module.exports = class ProcessingContext { reason: "The output band definitions are not properly given." }); } - else { - // ToDo: Send the following warning via subscriptions: - // "No bands are specified in the output parameter settings. The first band will be used for a gray-value visualisation." - visBands = [availableBands[0]]; - } + var region = Utils.bboxToGeoJson(bbox); dataCube.image().visualize({min: 0, max: 255, bands: visBands}).getThumbURL({ format: this.translateOutputFormat(format), dimensions: size, - region: bounds.bounds().getInfo() + region: region, +// crs: 'EPSG:3857' // toDo: Check results }, url => { if (typeof url !== 'string' || url.length === 0) { reject(new Errors.Internal({message: 'Download URL provided by Google Earth Engine is empty.'})); diff --git a/src/processgraph/datacube.js b/src/processgraph/datacube.js index 2a80b05..1e2bfc5 100644 --- a/src/processgraph/datacube.js +++ b/src/processgraph/datacube.js @@ -31,11 +31,13 @@ module.exports = class DataCube { this.data = data; } - computedObjectType(){ + computedObjectType() { + console.trace("Calling getInfo()"); + // ToDo: This is slow and needs to be replaced so that it uses a callback as parameter for getInfo() and the method will be async. return this.data.getInfo().type; } - objectType(){ + objectType() { if (this.data instanceof ee.Image){ return "Image"; } @@ -205,38 +207,24 @@ module.exports = class DataCube { getSpatialExtent() { var x = this.dimX(); var y = this.dimY(); - return { west: x.min(), east: x.max(), south: y.min(), - north: y.max() + north: y.max(), + crs: this.getCrs() }; } - getSpatialExtentAsGeeGeometry() { - var x = this.dimX(); - var y = this.dimY(); - if (x.crs() != y.crs()) { - throw "Spatial dimensions for x and y must not differ."; - } - return ee.Geometry.Rectangle([x.min(), y.min(), x.max(), y.max()], x.crs()); - } - getTemporalExtent() { return this.dimT(); } - getBands(force = false) { + getBands() { try { return this.dimBands().getValues(); } catch(e) { - if (force) { - return this.image().bandNames().getInfo(); - } - else { - return []; - } + return []; } } diff --git a/src/processgraph/processgraph.js b/src/processgraph/processgraph.js index d2b9cc0..12a7232 100644 --- a/src/processgraph/processgraph.js +++ b/src/processgraph/processgraph.js @@ -19,12 +19,7 @@ module.exports = class GeeProcessGraph extends ProcessGraph { // Optimization for web services to only load the extent of the data that is needed if no spatial extent is defined by the load_collection process if (this.loadCollectionRect && json.process_id === 'load_collection' && Utils.isObject(json.arguments) && !json.arguments.spatial_extent) { // ToDo: If an extent exists, use the intersecting area between tile and user-selected bounding box to further improve runtime. - json.arguments.spatial_extent = { - west: this.loadCollectionRect[0], - south: this.loadCollectionRect[1], - east: this.loadCollectionRect[2], - north: this.loadCollectionRect[3] - }; + json.arguments.spatial_extent = this.loadCollectionRect; } return new GeeProcessGraphNode(json, id, parent); } diff --git a/src/utils.js b/src/utils.js index b7c0e12..93365de 100644 --- a/src/utils.js +++ b/src/utils.js @@ -77,6 +77,19 @@ var Utils = { return datetime.replace(/\.\d{3}/, ''); // Remove milliseconds }, + bboxToGeoJson(bbox) { + return { + geodesic: false, + type: 'Polygon', + coordinates: + [ [ [ bbox.west, bbox.south ], + [ bbox.east, bbox.south ], + [ bbox.east, bbox.north ], + [ bbox.west, bbox.north ], + [ bbox.west, bbox.south ] ] ] + }; + }, + geoJsonBbox(geojson) { var getCoordinatesDump = function(gj) { switch(gj.type) { From e326b51567a33fbe36a5d2e0b14c076c05109573 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Fri, 6 Sep 2019 13:12:32 +0200 Subject: [PATCH 4/6] Improve error messages in services --- src/api/services.js | 2 +- src/errors.js | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/api/services.js b/src/api/services.js index 36268c9..fdfe9a6 100644 --- a/src/api/services.js +++ b/src/api/services.js @@ -52,7 +52,7 @@ module.exports = class ServicesAPI { }) .catch(e => next(Errors.wrap(e))); } catch(e) { - return next(e); + return next(Errors.wrap(e)); } }); } diff --git a/src/errors.js b/src/errors.js index 88f9bef..8613c2c 100644 --- a/src/errors.js +++ b/src/errors.js @@ -38,6 +38,9 @@ for(var name in errors) { else if (CommonUtils.isObject(obj)) { args = obj; } + else if (typeof obj === 'string') { + this.message = obj; + } this.info = args; this.message = CommonUtils.replacePlaceholders(this.message, this.info); }; From b2b0075ace468667f8ad81df27bcea2126642fb3 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Fri, 6 Sep 2019 13:52:09 +0200 Subject: [PATCH 5/6] Less error logging. --- src/processgraph/datacube.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/processgraph/datacube.js b/src/processgraph/datacube.js index 1e2bfc5..b98e40c 100644 --- a/src/processgraph/datacube.js +++ b/src/processgraph/datacube.js @@ -78,7 +78,7 @@ module.exports = class DataCube { else if (dataType === "ImageCollection") { // ToDo: Send warning via subscriptions if (global.server.serverContext.debug) { - console.warn("Compositing the image collection to a single image."); + console.log("Compositing the image collection to a single image."); } this.data = this.data.mosaic(); } From 345a5a0ddffc767cb6bfe38abb87e1152919f11b Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Tue, 10 Sep 2019 11:16:05 +0200 Subject: [PATCH 6/6] Updated load_collection spec for bands --- src/processes/load_collection.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/processes/load_collection.json b/src/processes/load_collection.json index 4357649..0c24ad0 100644 --- a/src/processes/load_collection.json +++ b/src/processes/load_collection.json @@ -125,7 +125,8 @@ { "type": "array", "items": { - "type": "string" + "type": "string", + "format": "band-name" } }, {