I have written a Parser that takes a JSON configuration and creates objects from it. I first create a well known object, and try to dynamically import a module (which may be from a user), while loading its class via the defined creator method of that module.
Here is some testing code:
import json
import imp
import os.path as path
from lib.config.members import Member
from lib.tasks.task import Task
class Parser(object):
def __init__(self):
self._loadedMods = {"tasks": {}}
def _load_module(self, clazz, modPart):
"""
imports and caches a module.
:param clazz: the filename of the module (i.e email, ping...)
:param modPart: the folder of the module. (i.e services, parsers...)
:return: the imported/cached module, or throws an error if it couldn't find it
"""
mods = self._loadedMods[modPart]
if clazz in mods:
return mods["class"]
else:
#mod = __import__(clazz)
p = path.join("lib", modPart, clazz + ".py")
mod = imp.load_source(clazz, p)
mods[clazz] = mod
return mod
def replace_with_import(self, objList, modPart, items_func, class_check):
"""
replaces configuration dicts with their objects by importing and creating it in the first step.
In the second step the original list of json config dicts gets replaced by the loaded objects
:param objList: the list of objects which is iterated on
:param modPart: the folder from the module (i.e tasks, parsers)
:param items_func: function to get a pointer on the list of json-config-objects to replace. Takes one argument and
should return a list of
:param class_check: currently unsupported
"""
for obj in objList:
repl = []
items = items_func(obj)
for clazzItem in items:
try:
clazz = clazzItem["class"]
mod = self._load_module(clazz, modPart)
item = mod.create(clazzItem)
if class_check(item):
repl.append(item)
else:
print " ignoring class " + clazzItem["class"] + "! It does not pass the class check!"
except ImportError, err:
print "could not import " + clazz + ": " + str(clazzItem) + "! reason:"
print str(err)
except KeyError, k:
print "Key " + str(k) + " not in classItem " + str(clazzItem)
except Exception, e:
print "Error while replacing class ( " + clazz + " :" + str(e) + ")"
del items[:]
items.extend(repl)
def _create_raw_Object(self, jsonDict, msgName, creator):
"""
creates an Main object from the configuration, but just parses raw data and hands it to the object
:param jsonDict: the configuration file part as dict
:param msgName: name of object for error message
:param creator: function pointer which is taking two arguments: identifier of the object and arguments.
:should return an object
:return: a list of objects returned by creator
"""
items = []
for key, val in jsonDict.items():
try:
item = creator(key, val)
items.append(item)
except Exception, e:
print "ignoring " + msgName + ": " + key + "! reason:"
print str(e)
return items
jsonFile = '''
{
"members":{
"homer":{
"name": "Homer Simpson",
"comment": "Security Inspector",
"tasks": [{"class":"email", "type": "donut", "args": {"rcpt": "homer_j_simpson@burnscorp.sp"}},
{"class":"email", "type": "do", "args": {"rcpt": "my_other_mail@burnscorp.sp"}}]
}
}
}
'''
jsonDict = json.loads(jsonFile)
parser = Parser()
creator = lambda name, values: Member(name, **values)
members = parser._create_raw_Object(jsonDict["members"], "Members", creator)
items_func = lambda member: member.get_tasks()
class_check = lambda task: isinstance(task, Task)
parser.replace_with_import(members, "tasks", items_func, class_check)
for d in members:
print d.__dict__
As you can see, a Member can have a list of arbitary tasks, and which one it should import is defined in its class attribute, but as soon as two of them has the same value for the class (which shouldn't break json the way we define it) I get a strange KeyError :
Key 'class' not in classItem {u'args': {u'rcpt': u'my_other_mail@burnscorp.sp'}, u'type': u'do', u'class': u'email'}
Why do I get this strange error? Any hint that my give me a clue whats going on is very welcome, as I feel hopeless, debugging this for hours.
I think that Member and Email/Task class are unrelated but Ill post them for completeness:
lib/config/members.py
class Member:
def __init__(self, id, name="", comment="", tasks=None):
self.id = id
self.name = name
self.tasks = []
self.add_task(tasks)
self.comment = comment
def get_id(self):
return self.id
def add_task(self, task):
if task is None:
return
if isinstance(task, list):
self.tasks.extend(task)
else:
self.tasks.append(task)
def get_tasks(self):
return self.tasks
lib/tasks/[task|email].py
class Task:
"""
Base class for all built-in Tasks.
"""
def set_task_type(self, taskType):
"""
sets the type of this task.
Be aware! this method can only get called once!
:param taskType: the type of this task
"""
if hasattr(self, "_taskType"):
raise Exception("taskType is only allowed to set once!")
self.taskType = taskType
def get_task_type(self):
"""
:return: the type set by set_type_task
"""
return self._taskType
"""
The email task.
"""
from lib.tasks.task import Task
class EmailTask(Task):
def __init__(self, **kwargs):
self.set_task_type(kwargs["type"])
self.recipient = kwargs["args"]["rcpt"]
def execute_task(self, msg):
pass
def create(taskDict):
return EmailTask(**taskDict)
It seems you are eating the actual exception by replacing it with your own custom print in replace_with_import. As I noted in the comment section.
You generally want to keep you try blocks small and very predictable, knowing exactly what can be raised and what you should handle at that point in the code. The less complexity in your try block the better.