pythondjangopython-3.xeclipsefunctools
functools stops working when switch to Python 3
We have migrated our Django project code base from Python 2.7 to 3.6 and suddenly what used to work stopped. Specifically, this:
map(functools.partial(self._assocUser, user=user), persistedGroupIds)
needed to be replaced with:
for group_id in persistedGroupIds:
self._assocUser(group_id, user)
and this:
persistedGroupIds = map(functools.partial(self._persistGroup, grp_mappings=attrAll.entitlements), saml_authorization_attributes)
needed to go to:
persistedGroupIds = []
for idp_group_name in saml_authorization_attributes:
persistedGroupIds.append(self._persistGroup(idp_group_name, attrAll.entitlements))
before the old functionality reappeared. Python 3's functools don't seem to work.
Here's the full listing of the code that works fine under Python 2:
from django.contrib.auth.models import User
from django.contrib.auth.models import Group
import functools
from mappings import SAMLAttributesConfig
from django.conf import settings
import logging
log = logging.getLogger(__name__)
class SAMLServiceProviderBackend(object):
empty_entitlements_message="IdP supplied incorrect authorization entitlements. Please contact their support."
def _assocUser(self, group_id, user):
group = Group.objects.get(id=group_id)
group.user_set.add(user)
return None
def _persistGroup(self,idp_group_name, grp_mappings):
group_name = grp_mappings[idp_group_name]
try:
group = Group.objects.get(name=group_name)
except Group.DoesNotExist:
group = Group(name=group_name)
group.save()
return group.id
def _extract_grp_entitlements(self,saml_authentication_attributes,groups):
result = []
input_length = len(saml_authentication_attributes[groups])
if input_length == 0:
log.error(self.empty_entitlements_message)
raise RuntimeError(self.empty_entitlements_message)
if input_length == 1:
result = [t.strip() for t in saml_authentication_attributes[groups][0].split(',')]
elif input_length:
result = saml_authentication_attributes[groups]
return result
# return [t.strip() for t in saml_authentication_attributes[groups][0].split(',')] \
# if len(saml_authentication_attributes[groups]) == 1\
# else saml_authentication_attributes[groups]
def authenticate(self, saml_authentication=None):
if not saml_authentication: # Using another authentication method
return None
attrAll = SAMLAttributesConfig(mappings_file_name=settings.AUTH_MAPPINGS_FILE).get_config()
groups = attrAll.entitlements.containerName
if saml_authentication.is_authenticated():
saml_authentication_attributes = saml_authentication.get_attributes()
saml_authorization_attributes = self._extract_grp_entitlements(saml_authentication_attributes,groups)
persistedGroupIds = map(functools.partial(self._persistGroup, grp_mappings=attrAll.entitlements), saml_authorization_attributes)
try:
user = User.objects.get(username=saml_authentication.get_nameid())
except User.DoesNotExist:
user = User(username=saml_authentication.get_nameid())
user.set_unusable_password()
try:
user.first_name = saml_authentication_attributes['samlNameId'][0]
except KeyError:
pass
try:
setattr(user, "first_name", saml_authentication_attributes[attrAll.subject.first_name][0])
except KeyError:
pass
#user.last_name = attributes['Last name'][0]
user.save()
map(functools.partial(self._assocUser, user=user), persistedGroupIds)
user.save()
return user
return None
def get_user(self, user_id):
try:
return User.objects.get(pk=user_id)
except User.DoesNotExist:
return None
The above code no longer works under Python 3 environment and only starts working to something like this, with a functools.partial() calls spelled out in a for loop:
from django.contrib.auth.models import User
from django.contrib.auth.models import Group
import functools
from .mappings import SAMLAttributesConfig
from django.conf import settings
import logging
log = logging.getLogger(__name__)
class SAMLServiceProviderBackend(object):
empty_entitlements_message="IdP supplied incorrect authorization entitlements. Please contact their support."
def _assocUser(self, group_id, user):
group = Group.objects.get(id=group_id)
group.user_set.add(user)
return None
def _persistGroup(self,idp_group_name, grp_mappings):
group_name = grp_mappings[idp_group_name]
try:
group = Group.objects.get(name=group_name)
except Group.DoesNotExist:
group = Group(name=group_name)
group.save()
return group.id
def _extract_grp_entitlements(self,saml_authentication_attributes,groups):
result = []
input_length = len(saml_authentication_attributes[groups])
if input_length == 0:
log.error(self.empty_entitlements_message)
raise RuntimeError(self.empty_entitlements_message)
if input_length == 1:
result = [t.strip() for t in saml_authentication_attributes[groups][0].split(',')]
elif input_length:
result = saml_authentication_attributes[groups]
return result
# return [t.strip() for t in saml_authentication_attributes[groups][0].split(',')] \
# if len(saml_authentication_attributes[groups]) == 1\
# else saml_authentication_attributes[groups]
def authenticate(self, saml_authentication=None):
if not saml_authentication: # Using another authentication method
return None
attrAll = SAMLAttributesConfig(mappings_file_name=settings.AUTH_MAPPINGS_FILE).get_config()
groups = attrAll.entitlements.containerName
if saml_authentication.is_authenticated():
saml_authentication_attributes = saml_authentication.get_attributes()
saml_authorization_attributes = self._extract_grp_entitlements(saml_authentication_attributes,groups)
persistedGroupIds = map(functools.partial(self._persistGroup, grp_mappings=attrAll.entitlements), saml_authorization_attributes)
try:
user = User.objects.get(username=saml_authentication.get_nameid())
except User.DoesNotExist:
user = User(username=saml_authentication.get_nameid())
user.set_unusable_password()
try:
user.first_name = saml_authentication_attributes['samlNameId'][0]
except KeyError:
pass
try:
setattr(user, "first_name", saml_authentication_attributes[attrAll.subject.first_name][0])
except KeyError:
pass
#user.last_name = attributes['Last name'][0]
user.save()
for group_id in persistedGroupIds:
self._assocUser(user = user, group_id = group_id)
# map(functools.partial(self._assocUser, user=user), persistedGroupIds)
user.save()
return user
return None
def get_user(self, user_id):
try:
return User.objects.get(pk=user_id)
except User.DoesNotExist:
return None
What can be wrong?
I'm using PyDev plugin in Eclipse. Here's how my Python interpreter is configured there:
Here's Eclipse's .pydevproject file:
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?eclipse-pydev version="1.0"?><pydev_project>
<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">venv3.6</pydev_property>
<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python interpreter</pydev_property>
<pydev_variables_property name="org.python.pydev.PROJECT_VARIABLE_SUBSTITUTION">
<key>DJANGO_SETTINGS_MODULE</key>
<value>reporting.settings</value>
<key>DJANGO_MANAGE_LOCATION</key>
<value>./manage.py</value>
<key>SAML_PLUGIN</key>
<value>/Users/sl/abc/venv3.6/lib/python3.6/site-packages/onelogin/saml2</value>
<key>PY</key>
<value>36</value>
</pydev_variables_property>
<pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
<path>/${PROJECT_DIR_NAME}</path>
</pydev_pathproperty>
<pydev_pathproperty name="org.python.pydev.PROJECT_EXTERNAL_SOURCE_PATH">
<path>${SAML_PLUGIN}</path>
</pydev_pathproperty>
</pydev_project>
Solution
In Python 3, the map function returns an iterator instead of a list.
This means that if you call map on a collection, the effects of the call are not materialised until you iterate over the resulting iterator.
Consider this class:
>>> class C:
... def __init__(self, x):
... self.x = x
... def double(self):
... self.x *= 2
... def __repr__(self):
... return '<C:{}>'.format(self.x)
...
Let's make a list of instances:
>>> cs = [C(x) for x in range(1, 4)]
>>> cs
[<C:1>, <C:2>, <C:3>]
Now use map to call each instance's double method:
>>> res = map(C.double, cs)
Note the result is not a list:
>>> res
<map object at 0x7ff276350470>
And the instances have not changed:
>>> cs
[<C:1>, <C:2>, <C:3>]
if we call next on the iterator, the instances are updated in turn.
>>> next(res)
>>> cs
[<C:2>, <C:2>, <C:3>]
>>> next(res)
>>> cs
[<C:2>, <C:4>, <C:3>]
>>> next(res)
>>> cs
[<C:2>, <C:4>, <C:6>]
In the code samples that you have provided, the result of calling map is not assigned to a variable, so map is being used for its side effects rather than its output. In Python 3 the correct way to do this is to loop over the iterable and call the function on each element:
>>> for c in cs:
c.double()
As the linked doc puts it:
Particularly tricky is
map()invoked for the side effects of the function; the correct transformation is to use a regular for loop (since creating a list would just be wasteful).