Skip to content

Commit

Permalink
Merge pull request #166 from eEcoLiDAR/development
Browse files Browse the repository at this point in the history
Development
  • Loading branch information
fnattino authored May 20, 2020
2 parents ffbe6d8 + 338d9a8 commit 469870a
Show file tree
Hide file tree
Showing 11 changed files with 122 additions and 67 deletions.
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## Unreleased

## 0.4.1 - 2020-05-20
## Added:
- select_equal filter accepts list of values to compare to the points' attributes
- also the attribute-based filter functions optionally return a mask to allow filter combinations

## Fixed:
- bug in writing/reading 'None' as parameter in the PLY comments

## 0.4.0 - 2020-05-13
## Added:
- build_volume module
Expand Down
4 changes: 2 additions & 2 deletions CITATION.cff
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ authors:
family-names: Koma
given-names: Zsófia
cff-version: "1.0.3"
date-released: 2020-05-13
date-released: 2020-05-20
doi: "10.5281/zenodo.1219422"
keywords:
- "airborne laser scanning"
Expand All @@ -62,5 +62,5 @@ keywords:
license: "Apache-2.0"
message: "If you use this software, please cite it using these metadata."
title: "Laserchicken: toolkit for ALS point clouds"
version: "0.4.0"
version: "0.4.1"
...
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ pip install laserchicken
* Create .zenodo.json file from CITATION.cff (using cffconvert)
```cffconvert --validate```
```cffconvert --ignore-suspect-keys --outputformat zenodo --outfile .zenodo.json```
* Set new version number in laserchicken/_version.py
* Set new version number in laserchicken/_version.txt
* Check that documentation uses the correct version
* Edit Changelog (based on commits in https://github.com/eecolidar/laserchicken/compare/v0.3.2...master)
* Test if package can be installed with pip (`pip install .`)
Expand Down
2 changes: 1 addition & 1 deletion laserchicken/_version.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.4.0
0.4.1
29 changes: 21 additions & 8 deletions laserchicken/filter.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,51 +16,64 @@
from laserchicken.utils import copy_point_cloud, add_metadata


def select_equal(point_cloud, attribute, value):
def select_equal(point_cloud, attribute, value, return_mask=False):
"""
Return the selection of the input point cloud that contains only points with a given attribute equal to some value.
If a list of values is given, select the points corresponding to any of the provided values.
:param point_cloud: Input point cloud.
:param attribute: The attribute name used for selection
:param value: The value to compare the attribute to
:return: A new point cloud containing only the selected points
:param value: The value(s) to compare the attribute to
:param return_mask: If true, return the mask corresponding to the selection
:return:
"""
_check_valid_arguments(attribute, point_cloud)
mask = point_cloud[point][attribute]['data'] == value
# broadcast using shape of the values
mask = point_cloud[point][attribute]['data'] == np.array(value)[..., None]
if mask.ndim > 1:
mask = np.any(mask, axis=0) # reduce
if return_mask:
return mask
point_cloud_filtered = copy_point_cloud(point_cloud, mask)
add_metadata(point_cloud_filtered, sys.modules[__name__],
{'attribute': attribute, 'value': value})
return point_cloud_filtered


def select_above(point_cloud, attribute, threshold):
def select_above(point_cloud, attribute, threshold, return_mask=False):
"""
Return the selection of the input point cloud that contains only points with a given attribute above some value.
:param point_cloud: Input point cloud
:param attribute: The attribute name used for selection
:param threshold: The threshold value used for selection
:return: A new point cloud containing only the selected points
:param return_mask: If true, return the mask corresponding to the selection
:return:
"""
_check_valid_arguments(attribute, point_cloud)
mask = point_cloud[point][attribute]['data'] > threshold
if return_mask:
return mask
point_cloud_filtered = copy_point_cloud(point_cloud, mask)
add_metadata(point_cloud_filtered, sys.modules[__name__],
{'attribute': attribute, 'threshold': threshold})
return point_cloud_filtered


def select_below(point_cloud, attribute, threshold):
def select_below(point_cloud, attribute, threshold, return_mask=False):
"""
Return the selection of the input point cloud that contains only points with a given attribute below some value.
:param point_cloud: Input point cloud
:param attribute: The attribute name used for selection
:param threshold: The threshold value used for selection
:return: A new point cloud containing only the selected points
:param return_mask: If true, return the mask corresponding to the selection
:return:
"""
_check_valid_arguments(attribute, point_cloud)
mask = point_cloud[point][attribute]['data'] < threshold
if return_mask:
return mask
point_cloud_filtered = copy_point_cloud(point_cloud, mask)
add_metadata(point_cloud_filtered, sys.modules[__name__],
{'attribute': attribute, 'threshold': threshold})
Expand Down
17 changes: 10 additions & 7 deletions laserchicken/io/ply_read.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import ast
import json
import numpy as np
from dateutil import parser

from json.decoder import JSONDecodeError
from struct import unpack, calcsize

from laserchicken.io.utils import convert_to_short_type, convert_to_single_character_type
Expand Down Expand Up @@ -78,12 +80,13 @@ def _read_header_line(ply, is_binary=False):

def _read_log(comments):
try:
log = ast.literal_eval(' '.join(comments)) if comments else []
except SyntaxError: # Log can't be read. Maybe a ply file with 'regular' comments and no log.
log = []
for i, entry in enumerate(log):
if 'time' in entry:
entry['time'] = parser.parse(entry['time'])
log = json.loads(' '.join(comments)) if comments else []
except JSONDecodeError:
try:
# legacy: comments for laserchicken < 0.4.0
log = ast.literal_eval(' '.join(comments)) if comments else []
except SyntaxError: # Log can't be read. Maybe a ply file with 'regular' comments and no log.
log = []
return log


Expand Down
23 changes: 1 addition & 22 deletions laserchicken/io/ply_write.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,31 +99,10 @@ def _write_comment(pc, ply):

head = 'comment [\n'
tail = 'comment ]\n'
formatted_entries = ',\n'.join(['comment ' + json.dumps(_stringify(entry), sort_keys=True) for entry in log]) + '\n'
formatted_entries = ',\n'.join(['comment ' + json.dumps(entry, sort_keys=True) for entry in log]) + '\n'
ply.write(head + formatted_entries + tail)


def _stringify(entry):
copy = {}
for key, value in _sort_by_key(entry):
if isinstance(value, dict):
copy[key] = _stringify(value)
elif isinstance(value, list):
copy[key] = [_stringify(entry) if isinstance(entry, dict) else entry for entry in value]
else:
if key == 'time':
copy[key] = str(value)
else:
copy[key] = value
return copy


def _sort_by_key(entry):
key_value_pairs = list(entry.items())
key_value_pairs.sort(key=lambda key_value_pair: key_value_pair[0])
return key_value_pairs


def _write_header_elements(pc, attributes, ply, element_name, get_num_elements=None):
if element_name in pc:
num_elements = get_num_elements(pc[element_name]) if get_num_elements else 1
Expand Down
7 changes: 5 additions & 2 deletions laserchicken/io/test_read_ply.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import dateutil
import os
import shutil
import unittest
Expand Down Expand Up @@ -89,8 +90,10 @@ def test_correctModulesLogged(self):
def test_correctTimesLogged(self):
log = load(self.test_file_path)['log']

self.assertListEqual([2018, 1, 18, 16, 1, 0, 3, 18, -1], list(log[0]['time'].timetuple()))
self.assertListEqual([2018, 1, 18, 16, 3, 0, 3, 18, -1], list(log[1]['time'].timetuple()))
self.assertListEqual([2018, 1, 18, 16, 1, 0, 3, 18, -1],
list(dateutil.parser.parse(log[0]['time']).timetuple()))
self.assertListEqual([2018, 1, 18, 16, 3, 0, 3, 18, -1],
list(dateutil.parser.parse(log[1]['time']).timetuple()))

def setUp(self):
os.mkdir(self._test_dir)
Expand Down
49 changes: 49 additions & 0 deletions laserchicken/test_filter.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,27 @@ def test_selectEqual_outputCorrect():
pc_out = select_equal(pc_in, 'return', 1)
assert_equal(len(pc_out[point]['x']['data']), 2)

@staticmethod
def test_selectEqual_multipleValues():
""" Correct number of results. """
pc_in = get_test_data()
pc_out = select_equal(pc_in, 'return', [1, 2])
assert_equal(len(pc_out[point]['x']['data']), 3)

@staticmethod
def test_selectEqual_maskCorrect():
""" Correct number of results. """
pc_in = get_test_data()
mask_out = select_equal(pc_in, 'return', 1, return_mask=True)
assert_equal(mask_out, np.array([1,1,0], dtype=bool))

@staticmethod
def test_selectEqual_maskEmpty():
""" Correct number of results. """
pc_in = get_test_data()
mask_out = select_equal(pc_in, 'return', 3, return_mask=True)
assert_equal(sum(mask_out), 0)


class TestSelectBelow(unittest.TestCase):
@staticmethod
Expand Down Expand Up @@ -88,6 +109,20 @@ def test_selectBelow_onlyOnePoint():
assert_almost_equal(pc_out[point]['y']['data'][0], 2.1)
assert_almost_equal(pc_out[point]['z']['data'][0], 3.1)
assert_almost_equal(pc_out[point]['return']['data'][0], 1)

@staticmethod
def test_selectBelow_maskCorrect():
""" Correct number of results. """
pc_in = get_test_data()
mask_out = select_below(pc_in, 'return', 2, return_mask=True)
assert_equal(mask_out, np.array([1,1,0], dtype=bool))

@staticmethod
def test_selectBelow_maskAll():
""" Correct number of results. """
pc_in = get_test_data()
mask_out = select_below(pc_in, 'return', 3, return_mask=True)
assert_equal(sum(mask_out), 3)


class TestSelectAbove(unittest.TestCase):
Expand Down Expand Up @@ -128,6 +163,20 @@ def test_selectBelow_onlyOnePoint():
assert_almost_equal(pc_out[point]['y']['data'][0], 2.3)
assert_almost_equal(pc_out[point]['z']['data'][0], 3.3)
assert_almost_equal(pc_out[point]['return']['data'][0], 2)

@staticmethod
def test_selectAbove_maskCorrect():
""" Correct number of results. """
pc_in = get_test_data()
mask_out = select_above(pc_in, 'return', 1, return_mask=True)
assert_equal(mask_out, np.array([0,0,1], dtype=bool))

@staticmethod
def test_selectAbove_maskAll():
""" Correct number of results. """
pc_in = get_test_data()
mask_out = select_above(pc_in, 'return', 0, return_mask=True)
assert_equal(sum(mask_out), 3)


class TestSelectPolygonWKT(unittest.TestCase):
Expand Down
2 changes: 1 addition & 1 deletion laserchicken/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ def add_metadata(point_cloud, module, params):
:param params:
:return:
"""
msg = {"time": datetime.datetime.utcnow(),
msg = {"time": str(datetime.datetime.utcnow()),
"module": module.__name__ if hasattr(module, "__name__") else str(module)}
if any(params):
msg["parameters"] = params
Expand Down
46 changes: 23 additions & 23 deletions tutorial.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -44,15 +44,15 @@
" -0.24100002, -0.24000002])},\n",
" 'raw_classification': {'type': 'uint8',\n",
" 'data': array([9, 9, 9, ..., 9, 9, 9], dtype=uint8)},\n",
" 'intensity': {'type': 'uint16',\n",
" 'data': array([ 41, 152, 12, ..., 10, 15, 10], dtype=uint16)},\n",
" 'gps_time': {'type': 'float64',\n",
" 'data': array([78563787.97322202, 78563787.93570042, 78563787.93571067, ...,\n",
" 78563778.28828931, 78563778.3107884 , 78563778.32578015])},\n",
" 'intensity': {'type': 'uint16',\n",
" 'data': array([ 41, 152, 12, ..., 10, 15, 10], dtype=uint16)}},\n",
" 'log': [{'time': datetime.datetime(2020, 3, 10, 8, 1, 8, 404699),\n",
" 78563778.28828931, 78563778.3107884 , 78563778.32578015])}},\n",
" 'log': [{'time': '2020-05-19 14:56:55.083339',\n",
" 'module': 'laserchicken.io.load',\n",
" 'parameters': {'path': 'testdata/AHN3.las', 'args': ()},\n",
" 'version': '0.3.2'}]}"
" 'version': '0.4.0'}]}"
]
},
"execution_count": 2,
Expand Down Expand Up @@ -91,21 +91,21 @@
" -0.24100002, -0.24000002])},\n",
" 'raw_classification': {'type': 'uint8',\n",
" 'data': array([9, 9, 9, ..., 9, 9, 9], dtype=uint8)},\n",
" 'intensity': {'type': 'uint16',\n",
" 'data': array([ 41, 152, 12, ..., 10, 15, 10], dtype=uint16)},\n",
" 'gps_time': {'type': 'float64',\n",
" 'data': array([78563787.97322202, 78563787.93570042, 78563787.93571067, ...,\n",
" 78563778.28828931, 78563778.3107884 , 78563778.32578015])},\n",
" 'intensity': {'type': 'uint16',\n",
" 'data': array([ 41, 152, 12, ..., 10, 15, 10], dtype=uint16)},\n",
" 'normalized_height': {'type': 'float64',\n",
" 'data': array([1.236, 1.311, 1.317, ..., 1.336, 1.336, 1.337])}},\n",
" 'log': [{'time': datetime.datetime(2020, 3, 10, 8, 1, 8, 404699),\n",
" 'log': [{'time': '2020-05-19 14:56:55.083339',\n",
" 'module': 'laserchicken.io.load',\n",
" 'parameters': {'path': 'testdata/AHN3.las', 'args': ()},\n",
" 'version': '0.3.2'},\n",
" {'time': datetime.datetime(2020, 3, 10, 8, 1, 8, 473466),\n",
" 'version': '0.4.0'},\n",
" {'time': '2020-05-19 14:56:55.149038',\n",
" 'module': 'laserchicken.normalize',\n",
" 'parameters': {'cell_size': None},\n",
" 'version': '0.3.2'}]}"
" 'version': '0.4.0'}]}"
]
},
"execution_count": 3,
Expand Down Expand Up @@ -137,21 +137,21 @@
" -0.24100002, -0.24000002])},\n",
" 'raw_classification': {'type': 'uint8',\n",
" 'data': array([9, 9, 9, ..., 9, 9, 9], dtype=uint8)},\n",
" 'intensity': {'type': 'uint16',\n",
" 'data': array([ 41, 152, 12, ..., 10, 15, 10], dtype=uint16)},\n",
" 'gps_time': {'type': 'float64',\n",
" 'data': array([78563787.97322202, 78563787.93570042, 78563787.93571067, ...,\n",
" 78563778.28828931, 78563778.3107884 , 78563778.32578015])},\n",
" 'intensity': {'type': 'uint16',\n",
" 'data': array([ 41, 152, 12, ..., 10, 15, 10], dtype=uint16)},\n",
" 'normalized_height': {'type': 'float64',\n",
" 'data': array([1.236, 1.311, 1.317, ..., 1.336, 1.336, 1.337])}},\n",
" 'log': [{'time': datetime.datetime(2020, 3, 10, 8, 1, 8, 404699),\n",
" 'log': [{'time': '2020-05-19 14:56:55.083339',\n",
" 'module': 'laserchicken.io.load',\n",
" 'parameters': {'path': 'testdata/AHN3.las', 'args': ()},\n",
" 'version': '0.3.2'},\n",
" {'time': datetime.datetime(2020, 3, 10, 8, 1, 8, 473466),\n",
" 'version': '0.4.0'},\n",
" {'time': '2020-05-19 14:56:55.149038',\n",
" 'module': 'laserchicken.normalize',\n",
" 'parameters': {'cell_size': None},\n",
" 'version': '0.3.2'}]}"
" 'version': '0.4.0'}]}"
]
},
"execution_count": 4,
Expand Down Expand Up @@ -245,13 +245,13 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Cylinder size in Bytes: 1214225560.6124551\n",
"Cylinder size in Bytes: 1262794583.0369534\n",
"Memory size in Bytes: 17179869184\n",
"Start tree creation\n",
"Done with env tree creation\n",
"Done with target tree creation\n",
"Extracting feature(s) \"['eigenv_1', 'eigenv_2', 'eigenv_3', 'normal_vector_1', 'normal_vector_2', 'normal_vector_3', 'slope']\"Extracting feature(s) \"['eigenv_1', 'eigenv_2', 'eigenv_3', 'normal_vector_1', 'normal_vector_2', 'normal_vector_3', 'slope']\" took 0.24 seconds\n",
"Extracting feature(s) \"['mean_z', 'std_z', 'coeff_var_z']\"Extracting feature(s) \"['mean_z', 'std_z', 'coeff_var_z']\" took 0.18 seconds\n",
"Extracting feature(s) \"['eigenv_1', 'eigenv_2', 'eigenv_3', 'normal_vector_1', 'normal_vector_2', 'normal_vector_3', 'slope']\"Extracting feature(s) \"['eigenv_1', 'eigenv_2', 'eigenv_3', 'normal_vector_1', 'normal_vector_2', 'normal_vector_3', 'slope']\" took 0.22 seconds\n",
"Extracting feature(s) \"['mean_z', 'std_z', 'coeff_var_z']\"Extracting feature(s) \"['mean_z', 'std_z', 'coeff_var_z']\" took 0.21 seconds\n",
"The following unrequested features were calculated as a side effect, but will not be returned: ['normal_vector_3', 'normal_vector_2', 'normal_vector_1', 'eigenv_3', 'eigenv_2', 'eigenv_1', 'coeff_var_z']\n"
]
}
Expand Down Expand Up @@ -574,12 +574,12 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Cylinder size in Bytes: 1214225560.6124551\n",
"Cylinder size in Bytes: 1262794583.0369534\n",
"Memory size in Bytes: 17179869184\n",
"Start tree creation\n",
"Done with env tree creation\n",
"Done with target tree creation\n",
"Extracting feature(s) \"['band_ratio_1<normalized_height<2']\"Extracting feature(s) \"['band_ratio_1<normalized_height<2']\" took 0.07 seconds\n"
"Extracting feature(s) \"['band_ratio_1<normalized_height<2']\"Extracting feature(s) \"['band_ratio_1<normalized_height<2']\" took 0.08 seconds\n"
]
}
],
Expand Down Expand Up @@ -623,7 +623,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.7"
"version": "3.8.2"
}
},
"nbformat": 4,
Expand Down

0 comments on commit 469870a

Please sign in to comment.