diff --git a/src/pg_to_evalscript/conversion.py b/src/pg_to_evalscript/conversion.py index 6179efb..a0a8315 100644 --- a/src/pg_to_evalscript/conversion.py +++ b/src/pg_to_evalscript/conversion.py @@ -86,13 +86,13 @@ def convert_from_process_graph( bands_dimension_name="bands", temporal_dimension_name="t", encode_result=True, - bands_metadata=[], + bands_metadata={}, ): all_nodes_valid, subgraphs = check_validity_and_subgraphs( process_graph, temporal_dimension_name, bands_dimension_name, user_defined_processes=user_defined_processes ) if all_nodes_valid: - nodes, input_bands, initial_data_name = generate_nodes_from_process_graph( + nodes, input_bands, initial_data_names = generate_nodes_from_process_graph( process_graph, bands_dimension_name, temporal_dimension_name, @@ -102,7 +102,7 @@ def convert_from_process_graph( evalscript = Evalscript( input_bands, nodes, - initial_data_name, + initial_data_names, n_output_bands=n_output_bands, sample_type=sample_type, units=units, @@ -165,8 +165,8 @@ def generate_nodes_from_process_graph( ) nodes = [] - input_bands = None - initial_data_name = None + input_bands = [] + initial_data_names = {} for node_id in execution_order: process_id = process_graph[node_id]["process_id"] @@ -174,8 +174,10 @@ def generate_nodes_from_process_graph( child_nodes = None if process_id == "load_collection": - input_bands = arguments.get("bands") + bands_for_datasource = arguments.get("bands") initial_data_name = "node_" + node_id + input_bands.append({"datasource": initial_data_name, "bands": bands_for_datasource}) + initial_data_names[node_id] = initial_data_name continue elif process_id == "save_result": continue @@ -257,4 +259,4 @@ def generate_nodes_from_process_graph( bands_dimension_name=bands_dimension_name, ) nodes.append(node) - return nodes, input_bands, initial_data_name + return nodes, input_bands, initial_data_names diff --git a/src/pg_to_evalscript/evalscript.py b/src/pg_to_evalscript/evalscript.py index be701d9..acf4ed3 100644 --- a/src/pg_to_evalscript/evalscript.py +++ b/src/pg_to_evalscript/evalscript.py @@ -20,7 +20,7 @@ def __init__( self, input_bands, nodes, - initial_data_name, + initial_data_names, n_output_bands=1, sample_type="AUTO", units=None, @@ -30,11 +30,11 @@ def __init__( datacube_definition_directory="javascript_datacube", output_dimensions=None, encode_result=True, - bands_metadata=[], + bands_metadata={}, ): self.input_bands = input_bands self.nodes = nodes - self.initial_data_name = initial_data_name + self.initial_data_names = initial_data_names self.n_output_bands = n_output_bands self.sample_type = sample_type self.units = units @@ -47,7 +47,7 @@ def __init__( self.bands_metadata = bands_metadata def write(self): - if self.input_bands is None: + if any(datasource_with_bands["bands"] is None for datasource_with_bands in self.input_bands): raise Exception("input_bands must be set!") newline = "\n" tab = "\t" @@ -55,7 +55,7 @@ def write(self): //VERSION=3 function setup() {{ return {{ - input: [{",".join([f"'{band}'" for band in self.input_bands])}], + input: [{",".join([f"{datasource_with_bands}" for datasource_with_bands in self.input_bands])}], output: {{ bands: {self.n_output_bands}, sampleType: "{self.sample_type}"{f", units: '{self.units}'" if self.units is not None else ''} }}, mosaicking: "{self.mosaicking}" }}; @@ -103,10 +103,24 @@ def write_ndarray_definition(self): return pkgutil.get_data("pg_to_evalscript", f"javascript_datacube/ndarray.js").decode("utf-8") def write_datacube_creation(self): - return f"let {self.initial_data_name} = new DataCube(samples, '{self.bands_dimension_name}', '{self.temporal_dimension_name}', true, {json.dumps(self.bands_metadata)}, scenes)" + datacube_creation = "" + if len(self.input_bands) > 1: + for datasource_with_bands in self.input_bands: + datasource_name = datasource_with_bands["datasource"] + datacube_creation += f"let {datasource_name} = new DataCube(samples.{datasource_name}, '{self.bands_dimension_name}', '{self.temporal_dimension_name}', true, {json.dumps(self.bands_metadata[datasource_name] if self.bands_metadata is not None and len(self.bands_metadata) > 0 else None)}, scenes)\n\t" + else: + datasource_with_bands = self.input_bands[0] + datasource_name = datasource_with_bands["datasource"] + datacube_creation += f"let {datasource_name} = new DataCube(samples, '{self.bands_dimension_name}', '{self.temporal_dimension_name}', true, {json.dumps(self.bands_metadata[datasource_name] if self.bands_metadata is not None and len(self.bands_metadata) > 0 else None)}, scenes)\n\t" + + return datacube_creation def write_runtime_global_constants(self): - return f"const INPUT_BANDS = {self.input_bands};" + all_bands = [] + for datasource_with_bands in self.input_bands: + all_bands.extend(datasource_with_bands["bands"]) + + return f"const INPUT_BANDS = {list(set(all_bands))};" def write_update_output(self): if self._output_dimensions is None: @@ -120,7 +134,13 @@ def write_update_output(self): size_without_original_temporal_dimensions = reduce( lambda x, y: x * y, sizes_without_original_temporal_dimensions, 1 ) - collection_scenes_length = "* collection.scenes.length" * number_of_original_temporal_dimensions + if len(self.input_bands) > 1: + collection_scenes_length = ( + "* Object.values(collection).reduce((acc, val) => acc + val.scenes.length, 0)" + * number_of_original_temporal_dimensions + ) + else: + collection_scenes_length = "* collection.scenes.length" * number_of_original_temporal_dimensions number_of_final_dimensions = len(self._output_dimensions) + 1 if self.encode_result else 0 return f""" function updateOutput(outputs, collection) {{ @@ -131,13 +151,18 @@ def write_update_output(self): def write_output_variable(self): if len(self.nodes) == 0: - return self.initial_data_name + return "_".join(self.initial_data_names.values()) return self.nodes[-1].node_varname_prefix + self.nodes[-1].node_id def determine_output_dimensions(self): dimensions_of_inputs_per_node = defaultdict(list) + all_bands = [] + for datasource_with_bands in self.input_bands: + if datasource_with_bands is not None and datasource_with_bands["bands"] is not None: + all_bands.extend(datasource_with_bands["bands"]) + initial_output_dimensions = [ - {"name": self.bands_dimension_name, "size": len(self.input_bands) if self.input_bands is not None else 0}, + {"name": self.bands_dimension_name, "size": len(set(all_bands)) if self.input_bands is not None else 0}, {"name": self.temporal_dimension_name, "size": None, "original_temporal": True}, ] @@ -147,7 +172,7 @@ def determine_output_dimensions(self): dimensions_of_inputs_per_node[self.nodes[0].node_id].append(initial_output_dimensions) for node in self.nodes: - output_dimensions = node.get_dimensions_change(dimensions_of_inputs_per_node[node.node_id]) + output_dimensions = node.get_dimensions_change(dimensions_of_inputs_per_node[node.node_id] if len(dimensions_of_inputs_per_node[node.node_id]) > 0 else [initial_output_dimensions]) for dependent in node.dependents: dimensions_of_inputs_per_node[dependent].append(output_dimensions) diff --git a/tests/integration_tests/fixtures.py b/tests/integration_tests/fixtures.py index afb090c..b275098 100644 --- a/tests/integration_tests/fixtures.py +++ b/tests/integration_tests/fixtures.py @@ -47,133 +47,135 @@ } -bands_metadata = [ - { - "name": "B01", - "common_name": "coastal", - "center_wavelength": 0.4427, - "full_width_half_max": 0.021, - "openeo:gsd": {"value": [60, 60], "unit": "m"}, - }, - { - "name": "B02", - "common_name": "blue", - "center_wavelength": 0.4924, - "full_width_half_max": 0.066, - "openeo:gsd": {"value": [10, 10], "unit": "m"}, - }, - { - "name": "B03", - "common_name": "green", - "center_wavelength": 0.5598, - "full_width_half_max": 0.036, - "openeo:gsd": {"value": [10, 10], "unit": "m"}, - }, - { - "name": "B04", - "common_name": "red", - "center_wavelength": 0.6646, - "full_width_half_max": 0.031, - "openeo:gsd": {"value": [10, 10], "unit": "m"}, - }, - { - "name": "B05", - "center_wavelength": 0.7041, - "full_width_half_max": 0.015, - "openeo:gsd": {"value": [20, 20], "unit": "m"}, - }, - { - "name": "B06", - "center_wavelength": 0.7405, - "full_width_half_max": 0.015, - "openeo:gsd": {"value": [20, 20], "unit": "m"}, - }, - { - "name": "B07", - "center_wavelength": 0.7828, - "full_width_half_max": 0.02, - "openeo:gsd": {"value": [20, 20], "unit": "m"}, - }, - { - "name": "B08", - "common_name": "nir", - "center_wavelength": 0.8328, - "full_width_half_max": 0.106, - "openeo:gsd": {"value": [10, 10], "unit": "m"}, - }, - { - "name": "B8A", - "common_name": "nir08", - "center_wavelength": 0.8647, - "full_width_half_max": 0.021, - "openeo:gsd": {"value": [20, 20], "unit": "m"}, - }, - { - "name": "B09", - "common_name": "nir09", - "center_wavelength": 0.9451, - "full_width_half_max": 0.02, - "openeo:gsd": {"value": [60, 60], "unit": "m"}, - }, - { - "name": "B11", - "common_name": "swir16", - "center_wavelength": 1.6137, - "full_width_half_max": 0.091, - "openeo:gsd": {"value": [20, 20], "unit": "m"}, - }, - { - "name": "B12", - "common_name": "swir22", - "center_wavelength": 2.2024, - "full_width_half_max": 0.175, - "openeo:gsd": {"value": [20, 20], "unit": "m"}, - }, - { - "description": "Aerosol Optical Thickness map, based on Sen2Cor processor", - "name": "AOT", - "openeo:gsd": {"value": [10, 10], "unit": "m"}, - }, - { - "description": "Scene classification data, based on Sen2Cor processor, codelist", - "name": "SCL", - "openeo:gsd": {"value": [20, 20], "unit": "m"}, - }, - { - "description": "Snow probability, based on Sen2Cor processor", - "name": "SNW", - "openeo:gsd": {"value": [20, 20], "unit": "m"}, - }, - { - "description": "Cloud probability, based on Sen2Cor processor", - "name": "CLD", - "openeo:gsd": {"value": [20, 20], "unit": "m"}, - }, - { - "description": "Cloud probability, based on s2cloudless", - "name": "CLP", - "openeo:gsd": {"value": [160, 160], "unit": "m"}, - }, - {"description": "Cloud masks", "name": "CLM", "openeo:gsd": {"value": [160, 160], "unit": "m"}}, - { - "description": "Sun azimuth angle", - "name": "sunAzimuthAngles", - "openeo:gsd": {"value": [5000, 5000], "unit": "m"}, - }, - { - "description": "Sun zenith angle", - "name": "sunZenithAngles", - "openeo:gsd": {"value": [5000, 5000], "unit": "m"}, - }, - { - "description": "Viewing azimuth angle", - "name": "viewAzimuthMean", - "openeo:gsd": {"value": [5000, 5000], "unit": "m"}, - }, - { - "description": "Viewing zenith angle", - "name": "viewZenithMean", - "openeo:gsd": {"value": [5000, 5000], "unit": "m"}, - }, - {"description": "The mask of data/no data pixels.", "name": "dataMask"}, -] \ No newline at end of file +bands_metadata = { + "node_loadcollection": [ + { + "name": "B01", + "common_name": "coastal", + "center_wavelength": 0.4427, + "full_width_half_max": 0.021, + "openeo:gsd": {"value": [60, 60], "unit": "m"}, + }, + { + "name": "B02", + "common_name": "blue", + "center_wavelength": 0.4924, + "full_width_half_max": 0.066, + "openeo:gsd": {"value": [10, 10], "unit": "m"}, + }, + { + "name": "B03", + "common_name": "green", + "center_wavelength": 0.5598, + "full_width_half_max": 0.036, + "openeo:gsd": {"value": [10, 10], "unit": "m"}, + }, + { + "name": "B04", + "common_name": "red", + "center_wavelength": 0.6646, + "full_width_half_max": 0.031, + "openeo:gsd": {"value": [10, 10], "unit": "m"}, + }, + { + "name": "B05", + "center_wavelength": 0.7041, + "full_width_half_max": 0.015, + "openeo:gsd": {"value": [20, 20], "unit": "m"}, + }, + { + "name": "B06", + "center_wavelength": 0.7405, + "full_width_half_max": 0.015, + "openeo:gsd": {"value": [20, 20], "unit": "m"}, + }, + { + "name": "B07", + "center_wavelength": 0.7828, + "full_width_half_max": 0.02, + "openeo:gsd": {"value": [20, 20], "unit": "m"}, + }, + { + "name": "B08", + "common_name": "nir", + "center_wavelength": 0.8328, + "full_width_half_max": 0.106, + "openeo:gsd": {"value": [10, 10], "unit": "m"}, + }, + { + "name": "B8A", + "common_name": "nir08", + "center_wavelength": 0.8647, + "full_width_half_max": 0.021, + "openeo:gsd": {"value": [20, 20], "unit": "m"}, + }, + { + "name": "B09", + "common_name": "nir09", + "center_wavelength": 0.9451, + "full_width_half_max": 0.02, + "openeo:gsd": {"value": [60, 60], "unit": "m"}, + }, + { + "name": "B11", + "common_name": "swir16", + "center_wavelength": 1.6137, + "full_width_half_max": 0.091, + "openeo:gsd": {"value": [20, 20], "unit": "m"}, + }, + { + "name": "B12", + "common_name": "swir22", + "center_wavelength": 2.2024, + "full_width_half_max": 0.175, + "openeo:gsd": {"value": [20, 20], "unit": "m"}, + }, + { + "description": "Aerosol Optical Thickness map, based on Sen2Cor processor", + "name": "AOT", + "openeo:gsd": {"value": [10, 10], "unit": "m"}, + }, + { + "description": "Scene classification data, based on Sen2Cor processor, codelist", + "name": "SCL", + "openeo:gsd": {"value": [20, 20], "unit": "m"}, + }, + { + "description": "Snow probability, based on Sen2Cor processor", + "name": "SNW", + "openeo:gsd": {"value": [20, 20], "unit": "m"}, + }, + { + "description": "Cloud probability, based on Sen2Cor processor", + "name": "CLD", + "openeo:gsd": {"value": [20, 20], "unit": "m"}, + }, + { + "description": "Cloud probability, based on s2cloudless", + "name": "CLP", + "openeo:gsd": {"value": [160, 160], "unit": "m"}, + }, + {"description": "Cloud masks", "name": "CLM", "openeo:gsd": {"value": [160, 160], "unit": "m"}}, + { + "description": "Sun azimuth angle", + "name": "sunAzimuthAngles", + "openeo:gsd": {"value": [5000, 5000], "unit": "m"}, + }, + { + "description": "Sun zenith angle", + "name": "sunZenithAngles", + "openeo:gsd": {"value": [5000, 5000], "unit": "m"}, + }, + { + "description": "Viewing azimuth angle", + "name": "viewAzimuthMean", + "openeo:gsd": {"value": [5000, 5000], "unit": "m"}, + }, + { + "description": "Viewing zenith angle", + "name": "viewZenithMean", + "openeo:gsd": {"value": [5000, 5000], "unit": "m"}, + }, + {"description": "The mask of data/no data pixels.", "name": "dataMask"}, + ] +} diff --git a/tests/integration_tests/test_integration.py b/tests/integration_tests/test_integration.py index fcfc07e..c2be441 100644 --- a/tests/integration_tests/test_integration.py +++ b/tests/integration_tests/test_integration.py @@ -274,7 +274,7 @@ def test_list_supported_processes(): @pytest.mark.parametrize( "new_bands", - [["B01", "B02", "B03"]], + [[{"datasource": "node_loadcollection", "bands": ["B01", "B02", "B03"]}]], ) def test_set_input_bands(new_bands): process_graph = get_process_graph_json("bands_null_graph") @@ -282,7 +282,7 @@ def test_set_input_bands(new_bands): result = convert_from_process_graph(process_graph, bands_dimension_name=bands_dimension_name, encode_result=False) evalscript = result[0]["evalscript"] - assert evalscript.input_bands is None + assert evalscript.input_bands[0]["bands"] is None assert ( evalscript._output_dimensions[0]["name"] == bands_dimension_name and evalscript._output_dimensions[0]["size"] == 0 @@ -297,7 +297,7 @@ def test_set_input_bands(new_bands): assert evalscript.input_bands == new_bands assert evalscript._output_dimensions[0]["name"] == bands_dimension_name and evalscript._output_dimensions[0][ "size" - ] == len(new_bands) + ] == len(new_bands[0]["bands"]) input_object = get_evalscript_input_object(evalscript.write()) assert input_object["input"] == new_bands diff --git a/tests/process_graphs/bands_null_graph.json b/tests/process_graphs/bands_null_graph.json index 060cde1..0467bcc 100644 --- a/tests/process_graphs/bands_null_graph.json +++ b/tests/process_graphs/bands_null_graph.json @@ -1,5 +1,5 @@ { - "loadco1":{ + "loadcollection":{ "process_id":"load_collection", "arguments":{ "id":"S2L1C", diff --git a/tests/process_graphs/test_graph_1.json b/tests/process_graphs/test_graph_1.json index d87cda2..29e038d 100644 --- a/tests/process_graphs/test_graph_1.json +++ b/tests/process_graphs/test_graph_1.json @@ -1,5 +1,5 @@ { - "1":{ + "loadcollection":{ "process_id":"load_collection", "arguments":{ "id":"BYOC", @@ -23,7 +23,7 @@ "process_id":"reduce_dimension", "arguments":{ "data":{ - "from_node":"1" + "from_node":"loadcollection" }, "dimension":"t", "reducer":{ diff --git a/tests/process_graphs/test_mean_ndvi.json b/tests/process_graphs/test_mean_ndvi.json index fa4483d..1d25e28 100644 --- a/tests/process_graphs/test_mean_ndvi.json +++ b/tests/process_graphs/test_mean_ndvi.json @@ -1,5 +1,5 @@ { - "1": { + "loadcollection": { "process_id": "load_collection", "arguments": { "bands": [ @@ -56,7 +56,7 @@ "process_id": "reduce_dimension", "arguments": { "data": { - "from_node": "1" + "from_node": "loadcollection" }, "reducer": { "process_graph": { diff --git a/tests/process_graphs/test_ndvi_s2l2a.json b/tests/process_graphs/test_ndvi_s2l2a.json index 2324779..9663a9c 100644 --- a/tests/process_graphs/test_ndvi_s2l2a.json +++ b/tests/process_graphs/test_ndvi_s2l2a.json @@ -1,5 +1,5 @@ { - "1":{ + "loadcollection":{ "process_id":"load_collection", "arguments":{ "id":"SENTINEL2_L2A_SENTINELHUB", @@ -33,7 +33,7 @@ "process_id":"ndvi", "arguments":{ "data":{ - "from_node":"1" + "from_node":"loadcollection" }, "target_band":"NDVI", "nir":"B08", diff --git a/tests/process_graphs/test_resample_cube_temporal_downsample.json b/tests/process_graphs/test_resample_cube_temporal_downsample.json index 5a0f1dc..38d733f 100644 --- a/tests/process_graphs/test_resample_cube_temporal_downsample.json +++ b/tests/process_graphs/test_resample_cube_temporal_downsample.json @@ -1,5 +1,5 @@ { - "loadco1": { + "loadcollection": { "process_id": "load_collection", "arguments": { "id": "S2L1C", @@ -22,7 +22,7 @@ "process_id": "apply_dimension", "arguments": { "data": { - "from_node": "loadco1" + "from_node": "loadcollection" }, "process": { "process_graph": { @@ -80,7 +80,7 @@ "process_id": "resample_cube_temporal", "arguments": { "data": { - "from_node": "loadco1" + "from_node": "loadcollection" }, "target": { "from_node": "rename_labels1" diff --git a/tests/process_graphs/test_resample_cube_temporal_upsample.json b/tests/process_graphs/test_resample_cube_temporal_upsample.json index 167302b..a65fa8d 100644 --- a/tests/process_graphs/test_resample_cube_temporal_upsample.json +++ b/tests/process_graphs/test_resample_cube_temporal_upsample.json @@ -1,5 +1,5 @@ { - "loadco1": { + "loadcollection": { "process_id": "load_collection", "arguments": { "id": "S2L1C", @@ -22,7 +22,7 @@ "process_id": "apply_dimension", "arguments": { "data": { - "from_node": "loadco1" + "from_node": "loadcollection" }, "process": { "process_graph": { @@ -83,7 +83,7 @@ "from_node": "rename_labels1" }, "target": { - "from_node": "loadco1" + "from_node": "loadcollection" }, "valid_within": 7 }