Skip to content

Commit

Permalink
Added logic to save and read whole object as single json
Browse files Browse the repository at this point in the history 
tendrl-spec: Tendrl/specifications#174
Signed-off-by: Shubhendu <shtripat@redhat.com>
  • Loading branch information
Shubhendu committed Jul 26, 2017
1 parent c5939c7 commit d297a31
Show file tree
Hide file tree
Showing 3 changed files with 26 additions and 180 deletions.
195 changes: 19 additions & 176 deletions tendrl/commons/objects/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,93 +75,26 @@ def load_definition(self):

def save(self, update=True, ttl=None):
self.render()
if "Message" not in self.__class__.__name__:
try:
# Generate current in memory object hash
self.hash = self._hash()
_hash_key = "/{0}/hash".format(self.value)
_stored_hash = None
try:
_stored_hash = NS._int.client.read(_hash_key).value
except (etcd.EtcdConnectionFailed, etcd.EtcdException) as ex:
if type(ex) != etcd.EtcdKeyNotFound:
NS._int.reconnect()
_stored_hash = NS._int.client.read(_hash_key).value
if self.hash == _stored_hash:
# No changes in stored object and current object,
# dont save current object to central store
if ttl:
etcd_utils.refresh(self.value, ttl)
return
except TypeError:
# no hash for this object, save the current hash as is
pass

if update:
current_obj = self.load()
for attr, val in vars(self).iteritems():
if isinstance(val, (types.FunctionType,
types.BuiltinFunctionType,
types.MethodType, types.BuiltinMethodType,
types.UnboundMethodType)) or \
attr.startswith("_") or attr in ['value', 'list']:
continue

if val is None and hasattr(current_obj, attr):
# if self.attr is None, use attr value from central
# store (i.e. current_obj.attr)
if getattr(current_obj, attr):
setattr(self, attr, getattr(current_obj, attr))
key = self.value + "/data"
try:
NS._int.wclient.write(key, self.json)
except (etcd.EtcdConnectionFailed, etcd.EtcdException):
NS._int.wreconnect()
NS._int.wclient.write(key, self.json)

self.updated_at = str(time_utils.now())
for item in self.render():
'''
Note: Log messages in this file have try-except
blocks to run
in the condition when the node_agent has not been
started and
name spaces are being created.
'''
try:
Event(
Message(
priority="debug",
publisher=NS.publisher_id,
payload={"message": "Writing %s to %s" %
(item['key'], item['value'])
}
)
)
except KeyError:
sys.stdout.write("Writing %s to %s" % (item['key'],
item['value']))
# convert list, dict (json) to python based on definitions
_type = self._defs.get("attrs", {}).get(item['name'],
{}).get("type")
if _type:
if _type.lower() in ['json', 'list']:
if item['value']:
try:
item['value'] = json.dumps(item['value'])
except ValueError as ex:
_msg = "Error save() attr %s for object %s" % \
(item['name'], self.__name__)
Event(
ExceptionMessage(
priority="debug",
publisher=NS.publisher_id,
payload={"message": _msg,
"exception": ex
}
)
)
try:
NS._int.wclient.write(item['key'], item['value'], quorum=True)
except (etcd.EtcdConnectionFailed, etcd.EtcdException):
NS._int.wreconnect()
NS._int.wclient.write(item['key'], item['value'], quorum=True)
if ttl:
etcd_utils.refresh(self.value, ttl)
def load(self):
self.render()
key = self.value + '/data'
val_str = NS._int.client.read(key).value
loc_dict = json.loads(val_str)
for attr_name, attr_val in vars(self).iteritems():
if not attr_name.startswith('_') and attr_name != "value":
_type = self._defs.get("attrs", {}).get(attr_name, {}).get("type")
if _type in ['json', 'list', 'Json', 'List', 'JSON', 'LIST']:
setattr(self, attr_name, json.loads(loc_dict[attr_name]))
else:
setattr(self, attr_name, loc_dict[attr_name])
return self

def load_all(self):
value = '/'.join(self.value.split('/')[:-1])
Expand All @@ -179,93 +112,6 @@ def load_all(self):
ins.append(self.load())
return ins

def load(self):
if "Message" not in self.__class__.__name__:
try:
# Generate current in memory object hash
self.hash = self._hash()
_hash_key = "/{0}/hash".format(self.value)
_stored_hash = None
try:
_stored_hash = NS._int.client.read(_hash_key).value
except (etcd.EtcdConnectionFailed, etcd.EtcdException) as ex:
if type(ex) != etcd.EtcdKeyNotFound:
NS._int.reconnect()
_stored_hash = NS._int.client.read(_hash_key).value
if self.hash == _stored_hash:
# No changes in stored object and current object,
# dont save current object to central store
return self
except TypeError:
# no hash for this object, save the current hash as is
pass

_copy = self._copy_vars()

for item in _copy.render():
try:
Event(
Message(
priority="debug",
publisher=NS.publisher_id,
payload={"message": "Reading %s" % item['key']}
)
)
except KeyError:
sys.stdout.write("Reading %s" % item['key'])

try:
etcd_resp = NS._int.client.read(item['key'], quorum=True)
except (etcd.EtcdConnectionFailed, etcd.EtcdException) as ex:
if type(ex) == etcd.EtcdKeyNotFound:
continue
else:
NS._int.reconnect()
etcd_resp = NS._int.client.read(item['key'], quorum=True)

value = etcd_resp.value
if item['dir']:
key = item['key'].split('/')[-1]
dct = dict(key=value)
if hasattr(_copy, item['name']):
dct = getattr(_copy, item['name'])
if type(dct) == dict:
dct[key] = value
else:
setattr(_copy, item['name'], dct)
else:
setattr(_copy, item['name'], dct)
continue

# convert list, dict (json) to python based on definitions
_type = self._defs.get("attrs", {}).get(item['name'],
{}).get("type")
if _type:
if _type.lower() in ['json', 'list']:
if value:
try:
value = json.loads(value.decode('utf-8'))
except ValueError as ex:
_msg = "Error load() attr %s for object %s" % \
(item['name'], self.__name__)
Event(
ExceptionMessage(
priority="debug",
publisher=NS.publisher_id,
payload={"message": _msg,
"exception": ex
}
)
)
else:
if _type.lower() == "list":
value = list()
if _type.lower() == "json":
value = dict()

setattr(_copy, item['name'], value)
return _copy

def exists(self):
self.render()
_exists = False
Expand All @@ -284,9 +130,6 @@ def _map_vars_to_tendrl_fields(self):
for attr, value in vars(self).iteritems():
_type = self._defs.get("attrs", {}).get(attr,
{}).get("type")
if _type:
_type = _type.lower()

if value is None:
value = ""
if attr.startswith("_") or attr in ['value', 'list']:
Expand Down
9 changes: 6 additions & 3 deletions tendrl/commons/objects/node_context/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,13 @@ def __init__(self, machine_id=None, node_id=None, fqdn=None,
self.node_id = node_id or self._get_node_id() or self._create_node_id()
self.fqdn = fqdn or socket.getfqdn()

curr_tags = []
curr_tags = ''
try:
curr_tags = NS._int.client.read("/nodes/%s/NodeContext/tags" %
self.node_id).value
nc_data = NS._int.client.read(
"/nodes/%s/NodeContext/data" %
self.node_id
).value
curr_tags = json.loads(nc_data)['tags']
except etcd.EtcdKeyNotFound:
pass

Expand Down
2 changes: 1 addition & 1 deletion tendrl/commons/utils/log_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ def log(log_priority, publisher_id, log_payload, job_id=None,
)
)
except Exception:
if log_priority.lower() == "error":
if log_priority in ["error", 'Error', 'ERROR']:
sys.stderr.write(log_payload.get("message"))
else:
sys.stdout.write(log_payload.get("message"))

0 comments on commit d297a31

Please sign in to comment.