From 0bd3351c00307e4f50aa267debc16ab3c321913c Mon Sep 17 00:00:00 2001 From: Jan Max Meyer Date: Mon, 25 Mar 2019 14:56:59 +0100 Subject: [PATCH 1/3] Proposal for an implementation of a recordBone recordBone can be used to store a data structure defined via a RelSkel within a skeleton. This enables to build more complex building blocks within a skeleton without adding a new module for that. --- bones/__init__.py | 1 + bones/recordBone.py | 299 +++++++++++++++++++++++++++++++++++++++++ render/html/default.py | 22 ++- render/json/default.py | 6 + 4 files changed, 325 insertions(+), 3 deletions(-) create mode 100644 bones/recordBone.py diff --git a/bones/__init__.py b/bones/__init__.py index 4886a20..4a89a4d 100644 --- a/bones/__init__.py +++ b/bones/__init__.py @@ -22,3 +22,4 @@ from server.bones.emailBone import emailBone from server.bones.randomSliceBone import randomSliceBone from server.bones.spatialBone import spatialBone +from server.bones.recordBone import recordBone diff --git a/bones/recordBone.py b/bones/recordBone.py new file mode 100644 index 0000000..0f0d38d --- /dev/null +++ b/bones/recordBone.py @@ -0,0 +1,299 @@ +# -*- coding: utf-8 -*- +from server.bones.bone import baseBone, getSystemInitialized +from server.errors import ReadFromClientError +import extjson + + +class recordBone(baseBone): + type = "record" + + def __init__(self, using, format=None, *args, **kwargs): + super(recordBone, self).__init__(*args, **kwargs) + + self.using = using + self.format = format + + if getSystemInitialized(): + self._usingSkelCache = using() + else: + self._usingSkelCache = None + + def setSystemInitialized(self): + super(recordBone, self).setSystemInitialized() + self._usingSkelCache = self.using() + + def __getattribute__(self, key): + # When no format was given, generate a format from the usingSkel. + if key == "format": + format = super(recordBone, self).__getattribute__(key) + if format: + return format + + if getSystemInitialized(): + return " ".join(["$(%s)" % k for k in self._usingSkelCache.keys()]) + + return super(recordBone, self).__getattribute__(key) + + def _restoreValueFromDatastore(self, val): + """ + Restores one of our values from the serialized data read from the datastore + + :param value: Json-Encoded datastore property + + :return: Our Value (with restored usingSkel data) + """ + value = extjson.loads(val) + assert isinstance(value, dict), "Read something from the datastore thats not a dict: %s" % str(type(value)) + + usingSkel = self._usingSkelCache + usingSkel.setValuesCache({}) + usingSkel.unserialize(value) + + return usingSkel.getValuesCache() + + def unserialize(self, valuesCache, name, expando): + if name not in expando: + valuesCache[name] = None + return True + + val = expando[name] + + if self.multiple: + valuesCache[name] = [] + + if not val: + return True + + if isinstance(val, list): + for res in val: + try: + valuesCache[name].append(self._restoreValueFromDatastore(res)) + except: + raise + pass + + else: + try: + valuesCache[name].append(self._restoreValueFromDatastore(val)) + except: + raise + pass + else: + valuesCache[name] = None + + if isinstance(val, list) and len(val) > 0: + try: + valuesCache[name] = self._restoreValueFromDatastore(val[0]) + except: + raise + pass + + else: + if val: + try: + valuesCache[name] = self._restoreValueFromDatastore(val) + except: + raise + pass + else: + valuesCache[name] = None + + return True + + def serialize(self, valuesCache, name, entity): + if not valuesCache[name]: + entity.set(name, None, False) + + if not self.multiple: + for k in entity.keys(): + if k.startswith("%s." % name): + del entity[k] + + else: + usingSkel = self._usingSkelCache + + if self.multiple: + res = [] + + for val in valuesCache[name]: + usingSkel.setValuesCache(val) + res.append(extjson.dumps(usingSkel.serialize())) + + entity.set(name, res, False) + + else: + usingSkel.setValuesCache(valuesCache[name]) + usingData = usingSkel.serialize() + + entity.set(name, extjson.dumps(usingData), False) + + # Copy attrs of our referenced entity in + if self.indexed: + for k, v in usingData.items(): + entity["%s.%s" % (name, k)] = v + + return entity + + def fromClient(self, valuesCache, name, data): + valuesCache[name] = [] + tmpRes = {} + + clientPrefix = "%s." % name + + for k, v in data.items(): + #print(k, v) + + if k.startswith(clientPrefix) or k == name: + if k == name: + k = k.replace(name, "", 1) + + else: + k = k.replace(clientPrefix, "", 1) + + if "." in k: + try: + idx, bname = k.split(".", 1) + idx = int(idx) + except ValueError: + idx = 0 + + try: + bname = k.split(".", 1) + except ValueError: + # We got some garbage as input; don't try to parse it + continue + + else: + idx = 0 + bname = k + + if not bname: + continue + + if not idx in tmpRes: + tmpRes[idx] = {} + + if bname in tmpRes[idx]: + if isinstance(tmpRes[idx][bname], list): + tmpRes[idx][bname].append(v) + else: + tmpRes[idx][bname] = [tmpRes[idx][bname], v] + else: + tmpRes[idx][bname] = v + + tmpList = [tmpRes[k] for k in sorted(tmpRes.keys())] + + errorDict = {} + forceFail = False + + if not tmpList and self.required: + return "No value selected!" + + for i, r in enumerate(tmpList[:]): + usingSkel = self._usingSkelCache + usingSkel.setValuesCache({}) + + if not usingSkel.fromClient(r): + for k, v in usingSkel.errors.items(): + errorDict["%s.%d.%s" % (name, i, k)] = v + forceFail = True + + tmpList[i] = usingSkel.getValuesCache() + + if self.multiple: + cleanList = [] + + for item in tmpList: + err = self.isInvalid(item) + if err: + errorDict["%s.%s" % (name, tmpList.index(item))] = err + else: + cleanList.append(item) + + if not cleanList: + errorDict[name] = "No value selected" + + valuesCache[name] = tmpList + else: + if tmpList: + val = tmpList[0] + else: + val = None + + err = self.isInvalid(val) + + if not err: + valuesCache[name] = val + if val is None: + errorDict[name] = "No value selected" + + if len(errorDict.keys()): + return ReadFromClientError(errorDict, forceFail) + + return None + + def getSearchTags(self, values, key): + def getValues(res, skel, valuesCache): + for k, bone in skel.items(): + if bone.searchable: + for tag in bone.getSearchTags(valuesCache, k): + if tag not in res: + res.append(tag) + return res + + value = values.get(key) + res = [] + + if not value: + return res + + if self.multiple: + for val in value: + res = getValues(res, self._usingSkelCache, val) + + else: + res = getValues(res, self._usingSkelCache, value) + + return res + + def getSearchDocumentFields(self, valuesCache, name, prefix=""): + def getValues(res, skel, valuesCache, searchPrefix): + for key, bone in skel.items(): + if bone.searchable: + res.extend(bone.getSearchDocumentFields(valuesCache, key, prefix=searchPrefix)) + + value = valuesCache.get(name) + res = [] + + if not value: + return res + + if self.multiple: + for idx, val in enumerate(value): + getValues(res, self._usingSkelCache, val, "%s%s_%s" % (prefix, name, str(idx))) + else: + getValues(res, self._usingSkelCache, value, "%s%s" % (prefix, name)) + + return res + + def getReferencedBlobs(self, valuesCache, name): + def blobsFromSkel(skel, valuesCache): + blobList = set() + for key, _bone in skel.items(): + blobList.update(_bone.getReferencedBlobs(valuesCache, key)) + return blobList + + res = set() + value = valuesCache.get(name) + + if not value: + return res + + if isinstance(value, list): + for val in value: + res.update(blobsFromSkel(self._usingSkelCache, val)) + + elif isinstance(value, dict): + res.update(blobsFromSkel(self._usingSkelCache, value)) + + return res diff --git a/render/html/default.py b/render/html/default.py index de80433..dff5ce0 100644 --- a/render/html/default.py +++ b/render/html/default.py @@ -313,7 +313,8 @@ def renderBoneValue(self, bone, skel, key): elif skelValue in bone.values: return KeyValueWrapper(skelValue, bone.values[skelValue]) return skelValue - elif bone.type=="relational" or bone.type.startswith("relational."): + + elif bone.type == "relational" or bone.type.startswith("relational."): if isinstance(skel[key], list): tmpList = [] for k in skel[key]: @@ -350,8 +351,23 @@ def renderBoneValue(self, bone, skel, key): "dest": self.collectSkelData(refSkel), "rel": usingData } - else: - return None + + elif bone.type == "record" or bone.type.startswith("record."): + usingSkel = bone._usingSkelCache + value = skel[key] + + if isinstance(value, list): + ret = [] + for entry in value: + usingSkel.setValuesCache(entry) + ret.append(self.collectSkelData(usingSkel)) + + return ret + + elif isinstance(value, dict): + usingSkel.setValuesCache(value) + return self.collectSkelData(usingSkel) + else: return skel[key] diff --git a/render/json/default.py b/render/json/default.py index d24941c..dd5aef7 100644 --- a/render/json/default.py +++ b/render/json/default.py @@ -54,6 +54,12 @@ def renderBoneStructure(self, bone): "relskel": self.renderSkelStructure(RefSkel.fromSkel(skeletonByKind(bone.kind), *bone.refKeys)) }) + elif bone.type == "record" or bone.type.startswith("record."): + ret.update({ + "multiple": bone.multiple, + "format": bone.format, + "using": self.renderSkelStructure(bone.using()) + }) elif bone.type == "select" or bone.type.startswith("select."): ret.update({ From 2e754b32ff6bdaecc41d58d0ab1f2a11e34d2147 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Steinr=C3=BCcken?= Date: Tue, 21 May 2019 10:36:19 +0200 Subject: [PATCH 2/3] Added basic constrains (multiple=True, indexed=False and a required formatString); removed Code that handled cases that are now excluded --- bones/recordBone.py | 130 +++++++++++--------------------------------- 1 file changed, 31 insertions(+), 99 deletions(-) diff --git a/bones/recordBone.py b/bones/recordBone.py index 0f0d38d..e0e5d50 100644 --- a/bones/recordBone.py +++ b/bones/recordBone.py @@ -7,11 +7,13 @@ class recordBone(baseBone): type = "record" - def __init__(self, using, format=None, *args, **kwargs): - super(recordBone, self).__init__(*args, **kwargs) + def __init__(self, using, format=None, indexed=False, multiple=True, *args, **kwargs): + super(recordBone, self).__init__(indexed=indexed, multiple=multiple, *args, **kwargs) self.using = using self.format = format + if not format or indexed or not multiple: + NotImplemented("A recordBone must not be indexed, must be multiple and must have a format set") if getSystemInitialized(): self._usingSkelCache = using() @@ -22,17 +24,6 @@ def setSystemInitialized(self): super(recordBone, self).setSystemInitialized() self._usingSkelCache = self.using() - def __getattribute__(self, key): - # When no format was given, generate a format from the usingSkel. - if key == "format": - format = super(recordBone, self).__getattribute__(key) - if format: - return format - - if getSystemInitialized(): - return " ".join(["$(%s)" % k for k in self._usingSkelCache.keys()]) - - return super(recordBone, self).__getattribute__(key) def _restoreValueFromDatastore(self, val): """ @@ -61,42 +52,21 @@ def unserialize(self, valuesCache, name, expando): if self.multiple: valuesCache[name] = [] - if not val: - return True - - if isinstance(val, list): - for res in val: - try: - valuesCache[name].append(self._restoreValueFromDatastore(res)) - except: - raise - pass + if not val: + return True - else: + if isinstance(val, list): + for res in val: try: - valuesCache[name].append(self._restoreValueFromDatastore(val)) + valuesCache[name].append(self._restoreValueFromDatastore(res)) except: raise - pass else: - valuesCache[name] = None - - if isinstance(val, list) and len(val) > 0: - try: - valuesCache[name] = self._restoreValueFromDatastore(val[0]) - except: - raise - pass + try: + valuesCache[name].append(self._restoreValueFromDatastore(val)) + except: + raise - else: - if val: - try: - valuesCache[name] = self._restoreValueFromDatastore(val) - except: - raise - pass - else: - valuesCache[name] = None return True @@ -104,33 +74,15 @@ def serialize(self, valuesCache, name, entity): if not valuesCache[name]: entity.set(name, None, False) - if not self.multiple: - for k in entity.keys(): - if k.startswith("%s." % name): - del entity[k] - else: usingSkel = self._usingSkelCache + res = [] - if self.multiple: - res = [] - - for val in valuesCache[name]: - usingSkel.setValuesCache(val) - res.append(extjson.dumps(usingSkel.serialize())) + for val in valuesCache[name]: + usingSkel.setValuesCache(val) + res.append(extjson.dumps(usingSkel.serialize())) - entity.set(name, res, False) - - else: - usingSkel.setValuesCache(valuesCache[name]) - usingData = usingSkel.serialize() - - entity.set(name, extjson.dumps(usingData), False) - - # Copy attrs of our referenced entity in - if self.indexed: - for k, v in usingData.items(): - entity["%s.%s" % (name, k)] = v + entity.set(name, res, False) return entity @@ -200,32 +152,19 @@ def fromClient(self, valuesCache, name, data): tmpList[i] = usingSkel.getValuesCache() - if self.multiple: - cleanList = [] - - for item in tmpList: - err = self.isInvalid(item) - if err: - errorDict["%s.%s" % (name, tmpList.index(item))] = err - else: - cleanList.append(item) - - if not cleanList: - errorDict[name] = "No value selected" + cleanList = [] - valuesCache[name] = tmpList - else: - if tmpList: - val = tmpList[0] + for item in tmpList: + err = self.isInvalid(item) + if err: + errorDict["%s.%s" % (name, tmpList.index(item))] = err else: - val = None + cleanList.append(item) - err = self.isInvalid(val) + if not cleanList: + errorDict[name] = "No value selected" - if not err: - valuesCache[name] = val - if val is None: - errorDict[name] = "No value selected" + valuesCache[name] = tmpList if len(errorDict.keys()): return ReadFromClientError(errorDict, forceFail) @@ -247,12 +186,8 @@ def getValues(res, skel, valuesCache): if not value: return res - if self.multiple: - for val in value: - res = getValues(res, self._usingSkelCache, val) - - else: - res = getValues(res, self._usingSkelCache, value) + for val in value: + res = getValues(res, self._usingSkelCache, val) return res @@ -268,11 +203,8 @@ def getValues(res, skel, valuesCache, searchPrefix): if not value: return res - if self.multiple: - for idx, val in enumerate(value): - getValues(res, self._usingSkelCache, val, "%s%s_%s" % (prefix, name, str(idx))) - else: - getValues(res, self._usingSkelCache, value, "%s%s" % (prefix, name)) + for idx, val in enumerate(value): + getValues(res, self._usingSkelCache, val, "%s%s_%s" % (prefix, name, str(idx))) return res From 8f98cc2fa1dcd993d26cb5fe7a2e75d68ca8df19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Steinr=C3=BCcken?= Date: Tue, 21 May 2019 10:46:34 +0200 Subject: [PATCH 3/3] Enhanced handling of non-required recordBones --- bones/recordBone.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bones/recordBone.py b/bones/recordBone.py index e0e5d50..0251d73 100644 --- a/bones/recordBone.py +++ b/bones/recordBone.py @@ -138,9 +138,6 @@ def fromClient(self, valuesCache, name, data): errorDict = {} forceFail = False - if not tmpList and self.required: - return "No value selected!" - for i, r in enumerate(tmpList[:]): usingSkel = self._usingSkelCache usingSkel.setValuesCache({}) @@ -161,11 +158,14 @@ def fromClient(self, valuesCache, name, data): else: cleanList.append(item) + valuesCache[name] = tmpList + if not cleanList: + if not (self.required or errorDict): + # Returning a error will only cause a warning if we are not required + return "No value selected!" errorDict[name] = "No value selected" - valuesCache[name] = tmpList - if len(errorDict.keys()): return ReadFromClientError(errorDict, forceFail)