From 4693b6198ab07bc74300391b61a5312a2faef63b Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Fri, 3 Nov 2023 10:41:38 +0100 Subject: [PATCH 01/23] Set multiple inputs and mark what still needs to be fixed --- src/pg_to_evalscript/conversion.py | 18 ++++++++------- src/pg_to_evalscript/evalscript.py | 22 +++++++++++-------- .../javascript_datacube/DataCube.js | 1 + 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/pg_to_evalscript/conversion.py b/src/pg_to_evalscript/conversion.py index 6179efb..471b36a 100644 --- a/src/pg_to_evalscript/conversion.py +++ b/src/pg_to_evalscript/conversion.py @@ -86,13 +86,13 @@ def convert_from_process_graph( bands_dimension_name="bands", temporal_dimension_name="t", encode_result=True, - bands_metadata=[], + bands_metadata={}, ): all_nodes_valid, subgraphs = check_validity_and_subgraphs( process_graph, temporal_dimension_name, bands_dimension_name, user_defined_processes=user_defined_processes ) if all_nodes_valid: - nodes, input_bands, initial_data_name = generate_nodes_from_process_graph( + nodes, input_bands, initial_data_names = generate_nodes_from_process_graph( process_graph, bands_dimension_name, temporal_dimension_name, @@ -102,7 +102,7 @@ def convert_from_process_graph( evalscript = Evalscript( input_bands, nodes, - initial_data_name, + initial_data_names, n_output_bands=n_output_bands, sample_type=sample_type, units=units, @@ -165,8 +165,8 @@ def generate_nodes_from_process_graph( ) nodes = [] - input_bands = None - initial_data_name = None + input_bands = [] + initial_data_names = {} for node_id in execution_order: process_id = process_graph[node_id]["process_id"] @@ -174,8 +174,10 @@ def generate_nodes_from_process_graph( child_nodes = None if process_id == "load_collection": - input_bands = arguments.get("bands") - initial_data_name = "node_" + node_id + bands_for_datasource = arguments.get("bands") + initial_data_name = "node_" + node_id + input_bands.append({"datasource": initial_data_name, "bands": bands_for_datasource}) + initial_data_names[node_id] = initial_data_name continue elif process_id == "save_result": continue @@ -257,4 +259,4 @@ def generate_nodes_from_process_graph( bands_dimension_name=bands_dimension_name, ) nodes.append(node) - return nodes, input_bands, initial_data_name + return nodes, input_bands, initial_data_names diff --git a/src/pg_to_evalscript/evalscript.py b/src/pg_to_evalscript/evalscript.py index be701d9..9ae7d35 100644 --- a/src/pg_to_evalscript/evalscript.py +++ b/src/pg_to_evalscript/evalscript.py @@ -20,7 +20,7 @@ def __init__( self, input_bands, nodes, - initial_data_name, + initial_data_names, n_output_bands=1, sample_type="AUTO", units=None, @@ -30,11 +30,11 @@ def __init__( datacube_definition_directory="javascript_datacube", output_dimensions=None, encode_result=True, - bands_metadata=[], + bands_metadata={}, ): self.input_bands = input_bands self.nodes = nodes - self.initial_data_name = initial_data_name + self.initial_data_names = initial_data_names self.n_output_bands = n_output_bands self.sample_type = sample_type self.units = units @@ -55,7 +55,7 @@ def write(self): //VERSION=3 function setup() {{ return {{ - input: [{",".join([f"'{band}'" for band in self.input_bands])}], + input: [{",".join([f"{datasource_with_bands}" for datasource_with_bands in self.input_bands])}], output: {{ bands: {self.n_output_bands}, sampleType: "{self.sample_type}"{f", units: '{self.units}'" if self.units is not None else ''} }}, mosaicking: "{self.mosaicking}" }}; @@ -103,10 +103,14 @@ def write_ndarray_definition(self): return pkgutil.get_data("pg_to_evalscript", f"javascript_datacube/ndarray.js").decode("utf-8") def write_datacube_creation(self): - return f"let {self.initial_data_name} = new DataCube(samples, '{self.bands_dimension_name}', '{self.temporal_dimension_name}', true, {json.dumps(self.bands_metadata)}, scenes)" + datacube_creation = "" + for node_id, bands_metadata_for_node in self.bands_metadata.items(): + datacube_creation += f"let {self.initial_data_names[node_id]} = new DataCube(samples, '{self.bands_dimension_name}', '{self.temporal_dimension_name}', true, {json.dumps(bands_metadata_for_node)}, scenes)\n\t" + return datacube_creation + def write_runtime_global_constants(self): - return f"const INPUT_BANDS = {self.input_bands};" + return f"const INPUT_BANDS = {self.input_bands};" # fix this def write_update_output(self): if self._output_dimensions is None: @@ -131,13 +135,13 @@ def write_update_output(self): def write_output_variable(self): if len(self.nodes) == 0: - return self.initial_data_name + return self.initial_data_names.values()[0] # ??? return self.nodes[-1].node_varname_prefix + self.nodes[-1].node_id def determine_output_dimensions(self): dimensions_of_inputs_per_node = defaultdict(list) initial_output_dimensions = [ - {"name": self.bands_dimension_name, "size": len(self.input_bands) if self.input_bands is not None else 0}, + {"name": self.bands_dimension_name, "size": len(self.input_bands) if self.input_bands is not None else 0}, # fix this {"name": self.temporal_dimension_name, "size": None, "original_temporal": True}, ] @@ -157,7 +161,7 @@ def set_output_dimensions(self, output_dimensions): self._output_dimensions = output_dimensions def set_input_bands(self, input_bands): - self.input_bands = input_bands + self.input_bands = input_bands # fix this when no input bands are set from driver's side output_dimensions = self.determine_output_dimensions() self.set_output_dimensions(output_dimensions) diff --git a/src/pg_to_evalscript/javascript_datacube/DataCube.js b/src/pg_to_evalscript/javascript_datacube/DataCube.js index 55886ac..7d020b5 100644 --- a/src/pg_to_evalscript/javascript_datacube/DataCube.js +++ b/src/pg_to_evalscript/javascript_datacube/DataCube.js @@ -73,6 +73,7 @@ class DataCube { makeArrayFromSamples(samples) { if (Array.isArray(samples)) { if (samples.length === 0) { + // fix this this._setDimensionLabelsIfEmpty(this.bands_dimension_name, INPUT_BANDS) // INPUT_BANDS is the default global var with all input band names return ndarray([], [0, 0]) } From 1a208646103549597018988368a2283574cc918f Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Wed, 8 Nov 2023 13:43:17 +0100 Subject: [PATCH 02/23] Fix stuff mostly related to input bands --- src/pg_to_evalscript/evalscript.py | 16 ++++++++++++---- .../javascript_datacube/DataCube.js | 1 - 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/pg_to_evalscript/evalscript.py b/src/pg_to_evalscript/evalscript.py index 9ae7d35..d2b8784 100644 --- a/src/pg_to_evalscript/evalscript.py +++ b/src/pg_to_evalscript/evalscript.py @@ -110,7 +110,11 @@ def write_datacube_creation(self): return datacube_creation def write_runtime_global_constants(self): - return f"const INPUT_BANDS = {self.input_bands};" # fix this + all_bands = [] + for datasource_with_bands in self.input_bands: + all_bands.extend(datasource_with_bands.bands) + + return f"const INPUT_BANDS = {list(set(all_bands))};" def write_update_output(self): if self._output_dimensions is None: @@ -135,13 +139,17 @@ def write_update_output(self): def write_output_variable(self): if len(self.nodes) == 0: - return self.initial_data_names.values()[0] # ??? + return self.initial_data_names.values()[0] # fix this ??? return self.nodes[-1].node_varname_prefix + self.nodes[-1].node_id def determine_output_dimensions(self): dimensions_of_inputs_per_node = defaultdict(list) + all_bands = [] + for datasource_with_bands in self.input_bands: + all_bands.extend(datasource_with_bands.bands) + initial_output_dimensions = [ - {"name": self.bands_dimension_name, "size": len(self.input_bands) if self.input_bands is not None else 0}, # fix this + {"name": self.bands_dimension_name, "size": len(set(all_bands)) if self.input_bands is not None else 0}, {"name": self.temporal_dimension_name, "size": None, "original_temporal": True}, ] @@ -161,7 +169,7 @@ def set_output_dimensions(self, output_dimensions): self._output_dimensions = output_dimensions def set_input_bands(self, input_bands): - self.input_bands = input_bands # fix this when no input bands are set from driver's side + self.input_bands = input_bands output_dimensions = self.determine_output_dimensions() self.set_output_dimensions(output_dimensions) diff --git a/src/pg_to_evalscript/javascript_datacube/DataCube.js b/src/pg_to_evalscript/javascript_datacube/DataCube.js index 7d020b5..55886ac 100644 --- a/src/pg_to_evalscript/javascript_datacube/DataCube.js +++ b/src/pg_to_evalscript/javascript_datacube/DataCube.js @@ -73,7 +73,6 @@ class DataCube { makeArrayFromSamples(samples) { if (Array.isArray(samples)) { if (samples.length === 0) { - // fix this this._setDimensionLabelsIfEmpty(this.bands_dimension_name, INPUT_BANDS) // INPUT_BANDS is the default global var with all input band names return ndarray([], [0, 0]) } From 1f00d938b2163ffb585a07600b14ba61d9b816d5 Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Wed, 8 Nov 2023 13:49:50 +0100 Subject: [PATCH 03/23] Fix names if length of nodes is 0 --- src/pg_to_evalscript/evalscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_to_evalscript/evalscript.py b/src/pg_to_evalscript/evalscript.py index d2b8784..43d17f2 100644 --- a/src/pg_to_evalscript/evalscript.py +++ b/src/pg_to_evalscript/evalscript.py @@ -139,7 +139,7 @@ def write_update_output(self): def write_output_variable(self): if len(self.nodes) == 0: - return self.initial_data_names.values()[0] # fix this ??? + return self.initial_data_names.values().join("_") return self.nodes[-1].node_varname_prefix + self.nodes[-1].node_id def determine_output_dimensions(self): From 1aca9e0867e8d91ed2a4dd9d0a65b631107c800c Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Wed, 8 Nov 2023 16:57:31 +0100 Subject: [PATCH 04/23] fix bug --- src/pg_to_evalscript/evalscript.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_to_evalscript/evalscript.py b/src/pg_to_evalscript/evalscript.py index 43d17f2..a26d6a2 100644 --- a/src/pg_to_evalscript/evalscript.py +++ b/src/pg_to_evalscript/evalscript.py @@ -112,7 +112,7 @@ def write_datacube_creation(self): def write_runtime_global_constants(self): all_bands = [] for datasource_with_bands in self.input_bands: - all_bands.extend(datasource_with_bands.bands) + all_bands.extend(datasource_with_bands["bands"]) return f"const INPUT_BANDS = {list(set(all_bands))};" @@ -146,7 +146,7 @@ def determine_output_dimensions(self): dimensions_of_inputs_per_node = defaultdict(list) all_bands = [] for datasource_with_bands in self.input_bands: - all_bands.extend(datasource_with_bands.bands) + all_bands.extend(datasource_with_bands["bands"]) initial_output_dimensions = [ {"name": self.bands_dimension_name, "size": len(set(all_bands)) if self.input_bands is not None else 0}, From 14a196eadd3d1b85073b99ec92fdd5771efd63ba Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Thu, 9 Nov 2023 14:30:11 +0100 Subject: [PATCH 05/23] add safety check for scenes which are undefined (probably wrong) --- src/pg_to_evalscript/evalscript.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pg_to_evalscript/evalscript.py b/src/pg_to_evalscript/evalscript.py index a26d6a2..7f9a2e0 100644 --- a/src/pg_to_evalscript/evalscript.py +++ b/src/pg_to_evalscript/evalscript.py @@ -130,8 +130,12 @@ def write_update_output(self): ) collection_scenes_length = "* collection.scenes.length" * number_of_original_temporal_dimensions number_of_final_dimensions = len(self._output_dimensions) + 1 if self.encode_result else 0 + # check that this is correct, how should the output be updated in this case or what is wrong that this doesn't execute (collection.scenes === undefined) return f""" function updateOutput(outputs, collection) {{ + if (!collection.scenes) {{ + return; + }} Object.values(outputs).forEach((output) => {{ output.bands = {number_of_final_dimensions} + {size_without_original_temporal_dimensions} {collection_scenes_length}; }}); From 8d75ab426c05cc02765f3e0e0534231cfaf76916 Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Thu, 9 Nov 2023 15:24:26 +0100 Subject: [PATCH 06/23] run linting --- src/pg_to_evalscript/conversion.py | 2 +- src/pg_to_evalscript/evalscript.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_to_evalscript/conversion.py b/src/pg_to_evalscript/conversion.py index 471b36a..a0a8315 100644 --- a/src/pg_to_evalscript/conversion.py +++ b/src/pg_to_evalscript/conversion.py @@ -175,7 +175,7 @@ def generate_nodes_from_process_graph( if process_id == "load_collection": bands_for_datasource = arguments.get("bands") - initial_data_name = "node_" + node_id + initial_data_name = "node_" + node_id input_bands.append({"datasource": initial_data_name, "bands": bands_for_datasource}) initial_data_names[node_id] = initial_data_name continue diff --git a/src/pg_to_evalscript/evalscript.py b/src/pg_to_evalscript/evalscript.py index 7f9a2e0..852a062 100644 --- a/src/pg_to_evalscript/evalscript.py +++ b/src/pg_to_evalscript/evalscript.py @@ -108,7 +108,7 @@ def write_datacube_creation(self): datacube_creation += f"let {self.initial_data_names[node_id]} = new DataCube(samples, '{self.bands_dimension_name}', '{self.temporal_dimension_name}', true, {json.dumps(bands_metadata_for_node)}, scenes)\n\t" return datacube_creation - + def write_runtime_global_constants(self): all_bands = [] for datasource_with_bands in self.input_bands: From e6774a327b12b5185e9dc63b234c802feb659e95 Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Fri, 10 Nov 2023 11:33:39 +0100 Subject: [PATCH 07/23] fix bug where there were no input bands set --- src/pg_to_evalscript/evalscript.py | 4 ++-- src/pg_to_evalscript/process_graph_utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_to_evalscript/evalscript.py b/src/pg_to_evalscript/evalscript.py index 852a062..7a95b57 100644 --- a/src/pg_to_evalscript/evalscript.py +++ b/src/pg_to_evalscript/evalscript.py @@ -47,7 +47,7 @@ def __init__( self.bands_metadata = bands_metadata def write(self): - if self.input_bands is None: + if any(datasource_with_bands["bands"] is None for datasource_with_bands in self.input_bands): raise Exception("input_bands must be set!") newline = "\n" tab = "\t" @@ -150,7 +150,7 @@ def determine_output_dimensions(self): dimensions_of_inputs_per_node = defaultdict(list) all_bands = [] for datasource_with_bands in self.input_bands: - all_bands.extend(datasource_with_bands["bands"]) + all_bands.extend(datasource_with_bands["bands"] if datasource_with_bands is not None and datasource_with_bands["bands"] is not None else []) initial_output_dimensions = [ {"name": self.bands_dimension_name, "size": len(set(all_bands)) if self.input_bands is not None else 0}, diff --git a/src/pg_to_evalscript/process_graph_utils.py b/src/pg_to_evalscript/process_graph_utils.py index 010a13b..ded7d2c 100644 --- a/src/pg_to_evalscript/process_graph_utils.py +++ b/src/pg_to_evalscript/process_graph_utils.py @@ -56,7 +56,7 @@ def get_dependents(dependencies): return dependents -def get_execution_order(dependencies, dependents): +def get_execution_order(dependencies, dependents): # fix this probably for more advanced PGs? entry_points = get_entry_points(dependencies) execution_order = [entry_point for entry_point in entry_points] remaining_nodes = set(dependencies.keys()).difference(execution_order) From 5b2655de71e20733bbadcdd0373aa09ecfccdcc4 Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Fri, 10 Nov 2023 16:33:08 +0100 Subject: [PATCH 08/23] add comment what to fix --- src/pg_to_evalscript/evalscript.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pg_to_evalscript/evalscript.py b/src/pg_to_evalscript/evalscript.py index 7a95b57..0a1055a 100644 --- a/src/pg_to_evalscript/evalscript.py +++ b/src/pg_to_evalscript/evalscript.py @@ -130,7 +130,7 @@ def write_update_output(self): ) collection_scenes_length = "* collection.scenes.length" * number_of_original_temporal_dimensions number_of_final_dimensions = len(self._output_dimensions) + 1 if self.encode_result else 0 - # check that this is correct, how should the output be updated in this case or what is wrong that this doesn't execute (collection.scenes === undefined) + # fix this (collection.scenes === undefined when multiple collections are used) https://docs.sentinel-hub.com/api/latest/evalscript/v3/#updateoutput-function-optional return f""" function updateOutput(outputs, collection) {{ if (!collection.scenes) {{ @@ -150,7 +150,11 @@ def determine_output_dimensions(self): dimensions_of_inputs_per_node = defaultdict(list) all_bands = [] for datasource_with_bands in self.input_bands: - all_bands.extend(datasource_with_bands["bands"] if datasource_with_bands is not None and datasource_with_bands["bands"] is not None else []) + all_bands.extend( + datasource_with_bands["bands"] + if datasource_with_bands is not None and datasource_with_bands["bands"] is not None + else [] + ) initial_output_dimensions = [ {"name": self.bands_dimension_name, "size": len(set(all_bands)) if self.input_bands is not None else 0}, From b67ec31bab2b38bec411168d0fc2cda5b801a25b Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Tue, 14 Nov 2023 10:53:47 +0100 Subject: [PATCH 09/23] fix updateOutput function if multiple load collections --- src/pg_to_evalscript/evalscript.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/pg_to_evalscript/evalscript.py b/src/pg_to_evalscript/evalscript.py index 0a1055a..a4f033d 100644 --- a/src/pg_to_evalscript/evalscript.py +++ b/src/pg_to_evalscript/evalscript.py @@ -128,14 +128,13 @@ def write_update_output(self): size_without_original_temporal_dimensions = reduce( lambda x, y: x * y, sizes_without_original_temporal_dimensions, 1 ) - collection_scenes_length = "* collection.scenes.length" * number_of_original_temporal_dimensions - number_of_final_dimensions = len(self._output_dimensions) + 1 if self.encode_result else 0 - # fix this (collection.scenes === undefined when multiple collections are used) https://docs.sentinel-hub.com/api/latest/evalscript/v3/#updateoutput-function-optional + if len(self.input_bands) > 1: + collection_scenes_length = "* Object.values(collection).reduce((acc, val) => acc + val.scenes.length, 0)" * number_of_original_temporal_dimensions + else: + collection_scenes_length = "* collection.scenes.length" * number_of_original_temporal_dimensions + number_of_final_dimensions = len(self._output_dimensions) + 1 if self.encode_result else 0 # 3 return f""" function updateOutput(outputs, collection) {{ - if (!collection.scenes) {{ - return; - }} Object.values(outputs).forEach((output) => {{ output.bands = {number_of_final_dimensions} + {size_without_original_temporal_dimensions} {collection_scenes_length}; }}); From a42c3244ece6cc3bdcf521968c466d48b789c28e Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Tue, 14 Nov 2023 13:42:33 +0100 Subject: [PATCH 10/23] run linting --- src/pg_to_evalscript/evalscript.py | 7 +++++-- src/pg_to_evalscript/process_graph_utils.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/pg_to_evalscript/evalscript.py b/src/pg_to_evalscript/evalscript.py index a4f033d..b6e35e8 100644 --- a/src/pg_to_evalscript/evalscript.py +++ b/src/pg_to_evalscript/evalscript.py @@ -129,10 +129,13 @@ def write_update_output(self): lambda x, y: x * y, sizes_without_original_temporal_dimensions, 1 ) if len(self.input_bands) > 1: - collection_scenes_length = "* Object.values(collection).reduce((acc, val) => acc + val.scenes.length, 0)" * number_of_original_temporal_dimensions + collection_scenes_length = ( + "* Object.values(collection).reduce((acc, val) => acc + val.scenes.length, 0)" + * number_of_original_temporal_dimensions + ) else: collection_scenes_length = "* collection.scenes.length" * number_of_original_temporal_dimensions - number_of_final_dimensions = len(self._output_dimensions) + 1 if self.encode_result else 0 # 3 + number_of_final_dimensions = len(self._output_dimensions) + 1 if self.encode_result else 0 # 3 return f""" function updateOutput(outputs, collection) {{ Object.values(outputs).forEach((output) => {{ diff --git a/src/pg_to_evalscript/process_graph_utils.py b/src/pg_to_evalscript/process_graph_utils.py index ded7d2c..ba4950c 100644 --- a/src/pg_to_evalscript/process_graph_utils.py +++ b/src/pg_to_evalscript/process_graph_utils.py @@ -56,7 +56,7 @@ def get_dependents(dependencies): return dependents -def get_execution_order(dependencies, dependents): # fix this probably for more advanced PGs? +def get_execution_order(dependencies, dependents): # fix this probably for more advanced PGs? entry_points = get_entry_points(dependencies) execution_order = [entry_point for entry_point in entry_points] remaining_nodes = set(dependencies.keys()).difference(execution_order) From 604de6af1d1a7d10518083226559798077050988 Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Wed, 15 Nov 2023 12:30:33 +0100 Subject: [PATCH 11/23] should work for both basic and fusion --- src/pg_to_evalscript/evalscript.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/pg_to_evalscript/evalscript.py b/src/pg_to_evalscript/evalscript.py index b6e35e8..c7ed7e3 100644 --- a/src/pg_to_evalscript/evalscript.py +++ b/src/pg_to_evalscript/evalscript.py @@ -104,9 +104,13 @@ def write_ndarray_definition(self): def write_datacube_creation(self): datacube_creation = "" - for node_id, bands_metadata_for_node in self.bands_metadata.items(): - datacube_creation += f"let {self.initial_data_names[node_id]} = new DataCube(samples, '{self.bands_dimension_name}', '{self.temporal_dimension_name}', true, {json.dumps(bands_metadata_for_node)}, scenes)\n\t" - + if len(self.bands_metadata) > 1: + for node_id, bands_metadata_for_node in self.bands_metadata.items(): + datacube_creation += f"let {self.initial_data_names[node_id]} = new DataCube(samples.node_{node_id}, '{self.bands_dimension_name}', '{self.temporal_dimension_name}', true, {json.dumps(bands_metadata_for_node)}, scenes)\n\t" + else: + for node_id, bands_metadata_for_node in self.bands_metadata.items(): + datacube_creation += f"let {self.initial_data_names[node_id]} = new DataCube(samples, '{self.bands_dimension_name}', '{self.temporal_dimension_name}', true, {json.dumps(bands_metadata_for_node)}, scenes)\n\t" + return datacube_creation def write_runtime_global_constants(self): From 40d55808281b445158d483cc5338250aff940f99 Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Wed, 15 Nov 2023 13:02:35 +0100 Subject: [PATCH 12/23] fix failing tests, so results match the processes https://processes.openeo.org/#order --- tests/unit_tests/test_order.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit_tests/test_order.py b/tests/unit_tests/test_order.py index 36ba3e5..df18439 100644 --- a/tests/unit_tests/test_order.py +++ b/tests/unit_tests/test_order.py @@ -18,14 +18,14 @@ def order_process_code(): ({"data": [6, -1, 2, 7, 4, 8, 3, 9, 9]}, [1, 2, 6, 4, 0, 3, 5, 7, 8]), ({"data": [6, -1, 2, 7, 4, 8, 3, 9, 9], "asc": False}, [7, 8, 5, 3, 0, 4, 6, 2, 1]), ({"data": [6, -1, 2, None, 7, 4, None, 8, 3, 9, 9]}, [1, 2, 8, 5, 0, 4, 7, 9, 10]), - ({"data": [6, -1, 2, None, 7, 4, None, 8, 3, 9, 9], "nodata": True}, [1, 2, 8, 5, 0, 4, 7, 9, 10, 6, 3]), + ({"data": [6, -1, 2, None, 7, 4, None, 8, 3, 9, 9], "nodata": True}, [1, 2, 8, 5, 0, 4, 7, 9, 10, 3, 6]), ( {"data": [6, -1, 2, None, 7, 4, None, 8, 3, 9, 9], "asc": False, "nodata": True}, - [10, 9, 7, 4, 0, 5, 8, 2, 1, 6, 3], + [9, 10, 7, 4, 0, 5, 8, 2, 1, 3, 6], ), ( {"data": [6, -1, 2, None, 7, 4, None, 8, 3, 9, 9], "asc": False, "nodata": False}, - [3, 6, 10, 9, 7, 4, 0, 5, 8, 2, 1], + [3, 6, 9, 10, 7, 4, 0, 5, 8, 2, 1], ), ({"data": ["2020-01-01T12:00:00Z", "2021-01-01T12:00:00Z", "2022-01-01T12:00:00Z"]}, [0, 1, 2]), ({"data": ["2022-01-01T12:00:00Z", "2021-01-01T12:00:00Z", "2020-01-01T12:00:00Z"]}, [2, 1, 0]), From c7bf92b9bc822e7fc904d206167d1aa47d7f8d75 Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Wed, 15 Nov 2023 14:01:32 +0100 Subject: [PATCH 13/23] fix wrong concatenation --- src/pg_to_evalscript/evalscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_to_evalscript/evalscript.py b/src/pg_to_evalscript/evalscript.py index c7ed7e3..456ccb1 100644 --- a/src/pg_to_evalscript/evalscript.py +++ b/src/pg_to_evalscript/evalscript.py @@ -149,7 +149,7 @@ def write_update_output(self): def write_output_variable(self): if len(self.nodes) == 0: - return self.initial_data_names.values().join("_") + return "_".join(self.initial_data_names.values()) return self.nodes[-1].node_varname_prefix + self.nodes[-1].node_id def determine_output_dimensions(self): From e40aaadc8d248d557c9e5f89a3c8fe88524af28c Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Wed, 15 Nov 2023 16:24:20 +0100 Subject: [PATCH 14/23] fix so correct samples are used for datacube creation --- src/pg_to_evalscript/evalscript.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/pg_to_evalscript/evalscript.py b/src/pg_to_evalscript/evalscript.py index 456ccb1..7636a51 100644 --- a/src/pg_to_evalscript/evalscript.py +++ b/src/pg_to_evalscript/evalscript.py @@ -104,13 +104,15 @@ def write_ndarray_definition(self): def write_datacube_creation(self): datacube_creation = "" - if len(self.bands_metadata) > 1: - for node_id, bands_metadata_for_node in self.bands_metadata.items(): - datacube_creation += f"let {self.initial_data_names[node_id]} = new DataCube(samples.node_{node_id}, '{self.bands_dimension_name}', '{self.temporal_dimension_name}', true, {json.dumps(bands_metadata_for_node)}, scenes)\n\t" - else: - for node_id, bands_metadata_for_node in self.bands_metadata.items(): - datacube_creation += f"let {self.initial_data_names[node_id]} = new DataCube(samples, '{self.bands_dimension_name}', '{self.temporal_dimension_name}', true, {json.dumps(bands_metadata_for_node)}, scenes)\n\t" - + if len(self.input_bands) > 1: + for datasource_with_bands in self.input_bands: + datasource_name = datasource_with_bands["datasource"] + datacube_creation += f"let {datasource_name} = new DataCube(samples.node_{datasource_name}, '{self.bands_dimension_name}', '{self.temporal_dimension_name}', true, {json.dumps(self.bands_metadata[datasource_name] if self.bands_metadata is not None and len(self.bands_metadata) > 0 else None)}, scenes)\n\t" + else: + datasource_with_bands = self.input_bands[0] + datasource_name = datasource_with_bands["datasource"] + datacube_creation += f"let {datasource_name} = new DataCube(samples, '{self.bands_dimension_name}', '{self.temporal_dimension_name}', true, {json.dumps(self.bands_metadata[datasource_name] if self.bands_metadata is not None and len(self.bands_metadata) > 0 else None)}, scenes)\n\t" + return datacube_creation def write_runtime_global_constants(self): From ee911c99fa8eb0cd76e66cf053586c729045c049 Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Wed, 15 Nov 2023 16:25:07 +0100 Subject: [PATCH 15/23] make bands_metadata use the new correct format --- tests/integration_tests/fixtures.py | 262 ++++++++++++++-------------- 1 file changed, 132 insertions(+), 130 deletions(-) diff --git a/tests/integration_tests/fixtures.py b/tests/integration_tests/fixtures.py index afb090c..3171916 100644 --- a/tests/integration_tests/fixtures.py +++ b/tests/integration_tests/fixtures.py @@ -47,133 +47,135 @@ } -bands_metadata = [ - { - "name": "B01", - "common_name": "coastal", - "center_wavelength": 0.4427, - "full_width_half_max": 0.021, - "openeo:gsd": {"value": [60, 60], "unit": "m"}, - }, - { - "name": "B02", - "common_name": "blue", - "center_wavelength": 0.4924, - "full_width_half_max": 0.066, - "openeo:gsd": {"value": [10, 10], "unit": "m"}, - }, - { - "name": "B03", - "common_name": "green", - "center_wavelength": 0.5598, - "full_width_half_max": 0.036, - "openeo:gsd": {"value": [10, 10], "unit": "m"}, - }, - { - "name": "B04", - "common_name": "red", - "center_wavelength": 0.6646, - "full_width_half_max": 0.031, - "openeo:gsd": {"value": [10, 10], "unit": "m"}, - }, - { - "name": "B05", - "center_wavelength": 0.7041, - "full_width_half_max": 0.015, - "openeo:gsd": {"value": [20, 20], "unit": "m"}, - }, - { - "name": "B06", - "center_wavelength": 0.7405, - "full_width_half_max": 0.015, - "openeo:gsd": {"value": [20, 20], "unit": "m"}, - }, - { - "name": "B07", - "center_wavelength": 0.7828, - "full_width_half_max": 0.02, - "openeo:gsd": {"value": [20, 20], "unit": "m"}, - }, - { - "name": "B08", - "common_name": "nir", - "center_wavelength": 0.8328, - "full_width_half_max": 0.106, - "openeo:gsd": {"value": [10, 10], "unit": "m"}, - }, - { - "name": "B8A", - "common_name": "nir08", - "center_wavelength": 0.8647, - "full_width_half_max": 0.021, - "openeo:gsd": {"value": [20, 20], "unit": "m"}, - }, - { - "name": "B09", - "common_name": "nir09", - "center_wavelength": 0.9451, - "full_width_half_max": 0.02, - "openeo:gsd": {"value": [60, 60], "unit": "m"}, - }, - { - "name": "B11", - "common_name": "swir16", - "center_wavelength": 1.6137, - "full_width_half_max": 0.091, - "openeo:gsd": {"value": [20, 20], "unit": "m"}, - }, - { - "name": "B12", - "common_name": "swir22", - "center_wavelength": 2.2024, - "full_width_half_max": 0.175, - "openeo:gsd": {"value": [20, 20], "unit": "m"}, - }, - { - "description": "Aerosol Optical Thickness map, based on Sen2Cor processor", - "name": "AOT", - "openeo:gsd": {"value": [10, 10], "unit": "m"}, - }, - { - "description": "Scene classification data, based on Sen2Cor processor, codelist", - "name": "SCL", - "openeo:gsd": {"value": [20, 20], "unit": "m"}, - }, - { - "description": "Snow probability, based on Sen2Cor processor", - "name": "SNW", - "openeo:gsd": {"value": [20, 20], "unit": "m"}, - }, - { - "description": "Cloud probability, based on Sen2Cor processor", - "name": "CLD", - "openeo:gsd": {"value": [20, 20], "unit": "m"}, - }, - { - "description": "Cloud probability, based on s2cloudless", - "name": "CLP", - "openeo:gsd": {"value": [160, 160], "unit": "m"}, - }, - {"description": "Cloud masks", "name": "CLM", "openeo:gsd": {"value": [160, 160], "unit": "m"}}, - { - "description": "Sun azimuth angle", - "name": "sunAzimuthAngles", - "openeo:gsd": {"value": [5000, 5000], "unit": "m"}, - }, - { - "description": "Sun zenith angle", - "name": "sunZenithAngles", - "openeo:gsd": {"value": [5000, 5000], "unit": "m"}, - }, - { - "description": "Viewing azimuth angle", - "name": "viewAzimuthMean", - "openeo:gsd": {"value": [5000, 5000], "unit": "m"}, - }, - { - "description": "Viewing zenith angle", - "name": "viewZenithMean", - "openeo:gsd": {"value": [5000, 5000], "unit": "m"}, - }, - {"description": "The mask of data/no data pixels.", "name": "dataMask"}, -] \ No newline at end of file +bands_metadata = { + "node_1": [ + { + "name": "B01", + "common_name": "coastal", + "center_wavelength": 0.4427, + "full_width_half_max": 0.021, + "openeo:gsd": {"value": [60, 60], "unit": "m"}, + }, + { + "name": "B02", + "common_name": "blue", + "center_wavelength": 0.4924, + "full_width_half_max": 0.066, + "openeo:gsd": {"value": [10, 10], "unit": "m"}, + }, + { + "name": "B03", + "common_name": "green", + "center_wavelength": 0.5598, + "full_width_half_max": 0.036, + "openeo:gsd": {"value": [10, 10], "unit": "m"}, + }, + { + "name": "B04", + "common_name": "red", + "center_wavelength": 0.6646, + "full_width_half_max": 0.031, + "openeo:gsd": {"value": [10, 10], "unit": "m"}, + }, + { + "name": "B05", + "center_wavelength": 0.7041, + "full_width_half_max": 0.015, + "openeo:gsd": {"value": [20, 20], "unit": "m"}, + }, + { + "name": "B06", + "center_wavelength": 0.7405, + "full_width_half_max": 0.015, + "openeo:gsd": {"value": [20, 20], "unit": "m"}, + }, + { + "name": "B07", + "center_wavelength": 0.7828, + "full_width_half_max": 0.02, + "openeo:gsd": {"value": [20, 20], "unit": "m"}, + }, + { + "name": "B08", + "common_name": "nir", + "center_wavelength": 0.8328, + "full_width_half_max": 0.106, + "openeo:gsd": {"value": [10, 10], "unit": "m"}, + }, + { + "name": "B8A", + "common_name": "nir08", + "center_wavelength": 0.8647, + "full_width_half_max": 0.021, + "openeo:gsd": {"value": [20, 20], "unit": "m"}, + }, + { + "name": "B09", + "common_name": "nir09", + "center_wavelength": 0.9451, + "full_width_half_max": 0.02, + "openeo:gsd": {"value": [60, 60], "unit": "m"}, + }, + { + "name": "B11", + "common_name": "swir16", + "center_wavelength": 1.6137, + "full_width_half_max": 0.091, + "openeo:gsd": {"value": [20, 20], "unit": "m"}, + }, + { + "name": "B12", + "common_name": "swir22", + "center_wavelength": 2.2024, + "full_width_half_max": 0.175, + "openeo:gsd": {"value": [20, 20], "unit": "m"}, + }, + { + "description": "Aerosol Optical Thickness map, based on Sen2Cor processor", + "name": "AOT", + "openeo:gsd": {"value": [10, 10], "unit": "m"}, + }, + { + "description": "Scene classification data, based on Sen2Cor processor, codelist", + "name": "SCL", + "openeo:gsd": {"value": [20, 20], "unit": "m"}, + }, + { + "description": "Snow probability, based on Sen2Cor processor", + "name": "SNW", + "openeo:gsd": {"value": [20, 20], "unit": "m"}, + }, + { + "description": "Cloud probability, based on Sen2Cor processor", + "name": "CLD", + "openeo:gsd": {"value": [20, 20], "unit": "m"}, + }, + { + "description": "Cloud probability, based on s2cloudless", + "name": "CLP", + "openeo:gsd": {"value": [160, 160], "unit": "m"}, + }, + {"description": "Cloud masks", "name": "CLM", "openeo:gsd": {"value": [160, 160], "unit": "m"}}, + { + "description": "Sun azimuth angle", + "name": "sunAzimuthAngles", + "openeo:gsd": {"value": [5000, 5000], "unit": "m"}, + }, + { + "description": "Sun zenith angle", + "name": "sunZenithAngles", + "openeo:gsd": {"value": [5000, 5000], "unit": "m"}, + }, + { + "description": "Viewing azimuth angle", + "name": "viewAzimuthMean", + "openeo:gsd": {"value": [5000, 5000], "unit": "m"}, + }, + { + "description": "Viewing zenith angle", + "name": "viewZenithMean", + "openeo:gsd": {"value": [5000, 5000], "unit": "m"}, + }, + {"description": "The mask of data/no data pixels.", "name": "dataMask"}, + ] +} From 3531d7b784c94407ce26ab6d070ee3a479e78f2c Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Wed, 15 Nov 2023 16:44:57 +0100 Subject: [PATCH 16/23] fix failing tests so they match new format of input bands --- tests/integration_tests/test_integration.py | 6 +++--- tests/process_graphs/bands_null_graph.json | 2 +- tests/process_graphs/test_filter_temporal.json | 4 ++-- .../test_resample_cube_temporal_downsample.json | 6 +++--- .../test_resample_cube_temporal_upsample.json | 6 +++--- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/integration_tests/test_integration.py b/tests/integration_tests/test_integration.py index fcfc07e..7899b01 100644 --- a/tests/integration_tests/test_integration.py +++ b/tests/integration_tests/test_integration.py @@ -274,7 +274,7 @@ def test_list_supported_processes(): @pytest.mark.parametrize( "new_bands", - [["B01", "B02", "B03"]], + [[{"datasource": "node_1", "bands": ["B01", "B02", "B03"]}]], ) def test_set_input_bands(new_bands): process_graph = get_process_graph_json("bands_null_graph") @@ -282,7 +282,7 @@ def test_set_input_bands(new_bands): result = convert_from_process_graph(process_graph, bands_dimension_name=bands_dimension_name, encode_result=False) evalscript = result[0]["evalscript"] - assert evalscript.input_bands is None + assert evalscript.input_bands[0]["bands"] is None assert ( evalscript._output_dimensions[0]["name"] == bands_dimension_name and evalscript._output_dimensions[0]["size"] == 0 @@ -297,7 +297,7 @@ def test_set_input_bands(new_bands): assert evalscript.input_bands == new_bands assert evalscript._output_dimensions[0]["name"] == bands_dimension_name and evalscript._output_dimensions[0][ "size" - ] == len(new_bands) + ] == len(new_bands[0]["bands"]) input_object = get_evalscript_input_object(evalscript.write()) assert input_object["input"] == new_bands diff --git a/tests/process_graphs/bands_null_graph.json b/tests/process_graphs/bands_null_graph.json index 060cde1..291a36b 100644 --- a/tests/process_graphs/bands_null_graph.json +++ b/tests/process_graphs/bands_null_graph.json @@ -1,5 +1,5 @@ { - "loadco1":{ + "1":{ "process_id":"load_collection", "arguments":{ "id":"S2L1C", diff --git a/tests/process_graphs/test_filter_temporal.json b/tests/process_graphs/test_filter_temporal.json index 0011a56..0a5a15b 100644 --- a/tests/process_graphs/test_filter_temporal.json +++ b/tests/process_graphs/test_filter_temporal.json @@ -1,5 +1,5 @@ { - "loadcollection": { + "1": { "process_id": "load_collection", "arguments": { "spatial_extent": null, @@ -12,7 +12,7 @@ "process_id": "filter_temporal", "arguments": { "data": { - "from_node": "loadcollection" + "from_node": "1" }, "extent": [ "2022-03-10T00:00:00.000Z", diff --git a/tests/process_graphs/test_resample_cube_temporal_downsample.json b/tests/process_graphs/test_resample_cube_temporal_downsample.json index 5a0f1dc..22f3b69 100644 --- a/tests/process_graphs/test_resample_cube_temporal_downsample.json +++ b/tests/process_graphs/test_resample_cube_temporal_downsample.json @@ -1,5 +1,5 @@ { - "loadco1": { + "1": { "process_id": "load_collection", "arguments": { "id": "S2L1C", @@ -22,7 +22,7 @@ "process_id": "apply_dimension", "arguments": { "data": { - "from_node": "loadco1" + "from_node": "1" }, "process": { "process_graph": { @@ -80,7 +80,7 @@ "process_id": "resample_cube_temporal", "arguments": { "data": { - "from_node": "loadco1" + "from_node": "1" }, "target": { "from_node": "rename_labels1" diff --git a/tests/process_graphs/test_resample_cube_temporal_upsample.json b/tests/process_graphs/test_resample_cube_temporal_upsample.json index 167302b..91958b6 100644 --- a/tests/process_graphs/test_resample_cube_temporal_upsample.json +++ b/tests/process_graphs/test_resample_cube_temporal_upsample.json @@ -1,5 +1,5 @@ { - "loadco1": { + "1": { "process_id": "load_collection", "arguments": { "id": "S2L1C", @@ -22,7 +22,7 @@ "process_id": "apply_dimension", "arguments": { "data": { - "from_node": "loadco1" + "from_node": "1" }, "process": { "process_graph": { @@ -83,7 +83,7 @@ "from_node": "rename_labels1" }, "target": { - "from_node": "loadco1" + "from_node": "1" }, "valid_within": 7 } From 0215634411d167b737eb7e883965701f7cc69d36 Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Wed, 15 Nov 2023 16:53:12 +0100 Subject: [PATCH 17/23] revert changes in order test --- tests/unit_tests/test_order.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit_tests/test_order.py b/tests/unit_tests/test_order.py index df18439..36ba3e5 100644 --- a/tests/unit_tests/test_order.py +++ b/tests/unit_tests/test_order.py @@ -18,14 +18,14 @@ def order_process_code(): ({"data": [6, -1, 2, 7, 4, 8, 3, 9, 9]}, [1, 2, 6, 4, 0, 3, 5, 7, 8]), ({"data": [6, -1, 2, 7, 4, 8, 3, 9, 9], "asc": False}, [7, 8, 5, 3, 0, 4, 6, 2, 1]), ({"data": [6, -1, 2, None, 7, 4, None, 8, 3, 9, 9]}, [1, 2, 8, 5, 0, 4, 7, 9, 10]), - ({"data": [6, -1, 2, None, 7, 4, None, 8, 3, 9, 9], "nodata": True}, [1, 2, 8, 5, 0, 4, 7, 9, 10, 3, 6]), + ({"data": [6, -1, 2, None, 7, 4, None, 8, 3, 9, 9], "nodata": True}, [1, 2, 8, 5, 0, 4, 7, 9, 10, 6, 3]), ( {"data": [6, -1, 2, None, 7, 4, None, 8, 3, 9, 9], "asc": False, "nodata": True}, - [9, 10, 7, 4, 0, 5, 8, 2, 1, 3, 6], + [10, 9, 7, 4, 0, 5, 8, 2, 1, 6, 3], ), ( {"data": [6, -1, 2, None, 7, 4, None, 8, 3, 9, 9], "asc": False, "nodata": False}, - [3, 6, 9, 10, 7, 4, 0, 5, 8, 2, 1], + [3, 6, 10, 9, 7, 4, 0, 5, 8, 2, 1], ), ({"data": ["2020-01-01T12:00:00Z", "2021-01-01T12:00:00Z", "2022-01-01T12:00:00Z"]}, [0, 1, 2]), ({"data": ["2022-01-01T12:00:00Z", "2021-01-01T12:00:00Z", "2020-01-01T12:00:00Z"]}, [2, 1, 0]), From d248eec4c79af2b2f540a417fb7023f3c907b2df Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Wed, 15 Nov 2023 17:22:21 +0100 Subject: [PATCH 18/23] fix bug --- src/pg_to_evalscript/evalscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_to_evalscript/evalscript.py b/src/pg_to_evalscript/evalscript.py index 7636a51..7a8cc77 100644 --- a/src/pg_to_evalscript/evalscript.py +++ b/src/pg_to_evalscript/evalscript.py @@ -107,7 +107,7 @@ def write_datacube_creation(self): if len(self.input_bands) > 1: for datasource_with_bands in self.input_bands: datasource_name = datasource_with_bands["datasource"] - datacube_creation += f"let {datasource_name} = new DataCube(samples.node_{datasource_name}, '{self.bands_dimension_name}', '{self.temporal_dimension_name}', true, {json.dumps(self.bands_metadata[datasource_name] if self.bands_metadata is not None and len(self.bands_metadata) > 0 else None)}, scenes)\n\t" + datacube_creation += f"let {datasource_name} = new DataCube(samples.{datasource_name}, '{self.bands_dimension_name}', '{self.temporal_dimension_name}', true, {json.dumps(self.bands_metadata[datasource_name] if self.bands_metadata is not None and len(self.bands_metadata) > 0 else None)}, scenes)\n\t" else: datasource_with_bands = self.input_bands[0] datasource_name = datasource_with_bands["datasource"] From f0c4f57bb69422e7ca63ce8470dd887cc65b2c4b Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Fri, 17 Nov 2023 14:11:31 +0100 Subject: [PATCH 19/23] remove comment as execution order is ok --- src/pg_to_evalscript/process_graph_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_to_evalscript/process_graph_utils.py b/src/pg_to_evalscript/process_graph_utils.py index ba4950c..010a13b 100644 --- a/src/pg_to_evalscript/process_graph_utils.py +++ b/src/pg_to_evalscript/process_graph_utils.py @@ -56,7 +56,7 @@ def get_dependents(dependencies): return dependents -def get_execution_order(dependencies, dependents): # fix this probably for more advanced PGs? +def get_execution_order(dependencies, dependents): entry_points = get_entry_points(dependencies) execution_order = [entry_point for entry_point in entry_points] remaining_nodes = set(dependencies.keys()).difference(execution_order) From 128d18d67679330abca9be0f3b5f580e9877992e Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Fri, 17 Nov 2023 15:40:04 +0100 Subject: [PATCH 20/23] fix for even more advanced PGs --- src/pg_to_evalscript/evalscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_to_evalscript/evalscript.py b/src/pg_to_evalscript/evalscript.py index 7a8cc77..347941d 100644 --- a/src/pg_to_evalscript/evalscript.py +++ b/src/pg_to_evalscript/evalscript.py @@ -175,7 +175,7 @@ def determine_output_dimensions(self): dimensions_of_inputs_per_node[self.nodes[0].node_id].append(initial_output_dimensions) for node in self.nodes: - output_dimensions = node.get_dimensions_change(dimensions_of_inputs_per_node[node.node_id]) + output_dimensions = node.get_dimensions_change(dimensions_of_inputs_per_node[node.node_id] if len(dimensions_of_inputs_per_node[node.node_id]) > 0 else [initial_output_dimensions]) for dependent in node.dependents: dimensions_of_inputs_per_node[dependent].append(output_dimensions) From c026d6186839b926dc02de81f8074c51446d2ef7 Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Wed, 22 Nov 2023 12:04:15 +0100 Subject: [PATCH 21/23] make code more readable --- src/pg_to_evalscript/evalscript.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/pg_to_evalscript/evalscript.py b/src/pg_to_evalscript/evalscript.py index 347941d..b20295c 100644 --- a/src/pg_to_evalscript/evalscript.py +++ b/src/pg_to_evalscript/evalscript.py @@ -158,11 +158,8 @@ def determine_output_dimensions(self): dimensions_of_inputs_per_node = defaultdict(list) all_bands = [] for datasource_with_bands in self.input_bands: - all_bands.extend( - datasource_with_bands["bands"] - if datasource_with_bands is not None and datasource_with_bands["bands"] is not None - else [] - ) + if datasource_with_bands is not None and datasource_with_bands["bands"] is not None: + all_bands.extend(datasource_with_bands["bands"]) initial_output_dimensions = [ {"name": self.bands_dimension_name, "size": len(set(all_bands)) if self.input_bands is not None else 0}, From ab606b63090230313478080f4c75fae75ce702ac Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Wed, 22 Nov 2023 16:39:25 +0100 Subject: [PATCH 22/23] rename load collection nodes in tests --- tests/integration_tests/fixtures.py | 2 +- tests/integration_tests/test_integration.py | 2 +- tests/process_graphs/bands_null_graph.json | 2 +- tests/process_graphs/test_filter_temporal.json | 4 ++-- tests/process_graphs/test_graph_1.json | 4 ++-- tests/process_graphs/test_mean_ndvi.json | 4 ++-- tests/process_graphs/test_ndvi_s2l2a.json | 4 ++-- .../test_resample_cube_temporal_downsample.json | 6 +++--- .../test_resample_cube_temporal_upsample.json | 6 +++--- 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/integration_tests/fixtures.py b/tests/integration_tests/fixtures.py index 3171916..b275098 100644 --- a/tests/integration_tests/fixtures.py +++ b/tests/integration_tests/fixtures.py @@ -48,7 +48,7 @@ bands_metadata = { - "node_1": [ + "node_loadcollection": [ { "name": "B01", "common_name": "coastal", diff --git a/tests/integration_tests/test_integration.py b/tests/integration_tests/test_integration.py index 7899b01..c2be441 100644 --- a/tests/integration_tests/test_integration.py +++ b/tests/integration_tests/test_integration.py @@ -274,7 +274,7 @@ def test_list_supported_processes(): @pytest.mark.parametrize( "new_bands", - [[{"datasource": "node_1", "bands": ["B01", "B02", "B03"]}]], + [[{"datasource": "node_loadcollection", "bands": ["B01", "B02", "B03"]}]], ) def test_set_input_bands(new_bands): process_graph = get_process_graph_json("bands_null_graph") diff --git a/tests/process_graphs/bands_null_graph.json b/tests/process_graphs/bands_null_graph.json index 291a36b..0467bcc 100644 --- a/tests/process_graphs/bands_null_graph.json +++ b/tests/process_graphs/bands_null_graph.json @@ -1,5 +1,5 @@ { - "1":{ + "loadcollection":{ "process_id":"load_collection", "arguments":{ "id":"S2L1C", diff --git a/tests/process_graphs/test_filter_temporal.json b/tests/process_graphs/test_filter_temporal.json index 0a5a15b..0011a56 100644 --- a/tests/process_graphs/test_filter_temporal.json +++ b/tests/process_graphs/test_filter_temporal.json @@ -1,5 +1,5 @@ { - "1": { + "loadcollection": { "process_id": "load_collection", "arguments": { "spatial_extent": null, @@ -12,7 +12,7 @@ "process_id": "filter_temporal", "arguments": { "data": { - "from_node": "1" + "from_node": "loadcollection" }, "extent": [ "2022-03-10T00:00:00.000Z", diff --git a/tests/process_graphs/test_graph_1.json b/tests/process_graphs/test_graph_1.json index d87cda2..29e038d 100644 --- a/tests/process_graphs/test_graph_1.json +++ b/tests/process_graphs/test_graph_1.json @@ -1,5 +1,5 @@ { - "1":{ + "loadcollection":{ "process_id":"load_collection", "arguments":{ "id":"BYOC", @@ -23,7 +23,7 @@ "process_id":"reduce_dimension", "arguments":{ "data":{ - "from_node":"1" + "from_node":"loadcollection" }, "dimension":"t", "reducer":{ diff --git a/tests/process_graphs/test_mean_ndvi.json b/tests/process_graphs/test_mean_ndvi.json index fa4483d..1d25e28 100644 --- a/tests/process_graphs/test_mean_ndvi.json +++ b/tests/process_graphs/test_mean_ndvi.json @@ -1,5 +1,5 @@ { - "1": { + "loadcollection": { "process_id": "load_collection", "arguments": { "bands": [ @@ -56,7 +56,7 @@ "process_id": "reduce_dimension", "arguments": { "data": { - "from_node": "1" + "from_node": "loadcollection" }, "reducer": { "process_graph": { diff --git a/tests/process_graphs/test_ndvi_s2l2a.json b/tests/process_graphs/test_ndvi_s2l2a.json index 2324779..9663a9c 100644 --- a/tests/process_graphs/test_ndvi_s2l2a.json +++ b/tests/process_graphs/test_ndvi_s2l2a.json @@ -1,5 +1,5 @@ { - "1":{ + "loadcollection":{ "process_id":"load_collection", "arguments":{ "id":"SENTINEL2_L2A_SENTINELHUB", @@ -33,7 +33,7 @@ "process_id":"ndvi", "arguments":{ "data":{ - "from_node":"1" + "from_node":"loadcollection" }, "target_band":"NDVI", "nir":"B08", diff --git a/tests/process_graphs/test_resample_cube_temporal_downsample.json b/tests/process_graphs/test_resample_cube_temporal_downsample.json index 22f3b69..38d733f 100644 --- a/tests/process_graphs/test_resample_cube_temporal_downsample.json +++ b/tests/process_graphs/test_resample_cube_temporal_downsample.json @@ -1,5 +1,5 @@ { - "1": { + "loadcollection": { "process_id": "load_collection", "arguments": { "id": "S2L1C", @@ -22,7 +22,7 @@ "process_id": "apply_dimension", "arguments": { "data": { - "from_node": "1" + "from_node": "loadcollection" }, "process": { "process_graph": { @@ -80,7 +80,7 @@ "process_id": "resample_cube_temporal", "arguments": { "data": { - "from_node": "1" + "from_node": "loadcollection" }, "target": { "from_node": "rename_labels1" diff --git a/tests/process_graphs/test_resample_cube_temporal_upsample.json b/tests/process_graphs/test_resample_cube_temporal_upsample.json index 91958b6..a65fa8d 100644 --- a/tests/process_graphs/test_resample_cube_temporal_upsample.json +++ b/tests/process_graphs/test_resample_cube_temporal_upsample.json @@ -1,5 +1,5 @@ { - "1": { + "loadcollection": { "process_id": "load_collection", "arguments": { "id": "S2L1C", @@ -22,7 +22,7 @@ "process_id": "apply_dimension", "arguments": { "data": { - "from_node": "1" + "from_node": "loadcollection" }, "process": { "process_graph": { @@ -83,7 +83,7 @@ "from_node": "rename_labels1" }, "target": { - "from_node": "1" + "from_node": "loadcollection" }, "valid_within": 7 } From 78947f718c0e86025873c8641aafbef9215d21f9 Mon Sep 17 00:00:00 2001 From: Zan Pecovnik Date: Wed, 22 Nov 2023 16:40:14 +0100 Subject: [PATCH 23/23] remove comment --- src/pg_to_evalscript/evalscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_to_evalscript/evalscript.py b/src/pg_to_evalscript/evalscript.py index b20295c..acf4ed3 100644 --- a/src/pg_to_evalscript/evalscript.py +++ b/src/pg_to_evalscript/evalscript.py @@ -141,7 +141,7 @@ def write_update_output(self): ) else: collection_scenes_length = "* collection.scenes.length" * number_of_original_temporal_dimensions - number_of_final_dimensions = len(self._output_dimensions) + 1 if self.encode_result else 0 # 3 + number_of_final_dimensions = len(self._output_dimensions) + 1 if self.encode_result else 0 return f""" function updateOutput(outputs, collection) {{ Object.values(outputs).forEach((output) => {{