#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 2008-2016 California Institute of Technology.
# Copyright (c) 2016-2023 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - https://github.com/uqfoundation/dill/blob/master/LICENSE
"""
Methods for detecting objects leading to pickling failures.
"""
import dis
from inspect import ismethod, isfunction, istraceback, isframe, iscode
from .pointers import parent, reference, at, parents, children
from .logger import trace
__all__ = ['baditems','badobjects','badtypes','code','errors','freevars',
'getmodule','globalvars','nestedcode','nestedglobals','outermost',
'referredglobals','referrednested','trace','varnames']
def getmodule(object, _filename=None, force=False):
"""get the module of the object"""
from inspect import getmodule as getmod
module = getmod(object, _filename)
if module or not force: return module
import builtins
from .source import getname
name = getname(object, force=True)
return builtins if name in vars(builtins).keys() else None
def outermost(func): # is analogous to getsource(func,enclosing=True)
"""get outermost enclosing object (i.e. the outer function in a closure)
NOTE: this is the object-equivalent of getsource(func, enclosing=True)
"""
if ismethod(func):
_globals = func.__func__.__globals__ or {}
elif isfunction(func):
_globals = func.__globals__ or {}
else:
return #XXX: or raise? no matches
_globals = _globals.items()
# get the enclosing source
from .source import getsourcelines
try: lines,lnum = getsourcelines(func, enclosing=True)
except Exception: #TypeError, IOError
lines,lnum = [],None
code = ''.join(lines)
# get all possible names,objects that are named in the enclosing source
_locals = ((name,obj) for (name,obj) in _globals if name in code)
# now only save the objects that generate the enclosing block
for name,obj in _locals: #XXX: don't really need 'name'
try:
if getsourcelines(obj) == (lines,lnum): return obj
except Exception: #TypeError, IOError
pass
return #XXX: or raise? no matches
def nestedcode(func, recurse=True): #XXX: or return dict of {co_name: co} ?
"""get the code objects for any nested functions (e.g. in a closure)"""
func = code(func)
if not iscode(func): return [] #XXX: or raise? no matches
nested = set()
for co in func.co_consts:
if co is None: continue
co = code(co)
if co:
nested.add(co)
if recurse: nested |= set(nestedcode(co, recurse=True))
return list(nested)
def code(func):
"""get the code object for the given function or method
NOTE: use dill.source.getsource(CODEOBJ) to get the source code
"""
if ismethod(func): func = func.__func__
if isfunction(func): func = func.__code__
if istraceback(func): func = func.tb_frame
if isframe(func): func = func.f_code
if iscode(func): return func
return
#XXX: ugly: parse dis.dis for name after "<code object" in line and in globals?
def referrednested(func, recurse=True): #XXX: return dict of {__name__: obj} ?
"""get functions defined inside of func (e.g. inner functions in a closure)
NOTE: results may differ if the function has been executed or not.
If len(nestedcode(func)) > len(referrednested(func)), try calling func().
If possible, python builds code objects, but delays building functions
until func() is called.
"""
import gc
funcs = set()
# get the code objects, and try to track down by referrence
for co in nestedcode(func, recurse):
# look for function objects that refer to the code object
for obj in gc.get_referrers(co):
# get methods
_ = getattr(obj, '__func__', None) # ismethod
if getattr(_, '__code__', None) is co: funcs.add(obj)
# get functions
elif getattr(obj, '__code__', None) is co: funcs.add(obj)
# get frame objects
elif getattr(obj, 'f_code', None) is co: funcs.add(obj)
# get code objects
elif hasattr(obj, 'co_code') and obj is co: funcs.add(obj)
# frameobjs => func.__code__.co_varnames not in func.__code__.co_cellvars
# funcobjs => func.__code__.co_cellvars not in func.__code__.co_varnames
# frameobjs are not found, however funcobjs are...
# (see: test_mixins.quad ... and test_mixins.wtf)
# after execution, code objects get compiled, and then may be found by gc
return list(funcs)
def freevars(func):
"""get objects defined in enclosing code that are referred to by func
returns a dict of {name:object}"""
if ismethod(func): func = func.__func__
if isfunction(func):
closures = func.__closure__ or ()
func = func.__code__.co_freevars # get freevars
else:
return {}
def get_cell_contents():
for name, c in zip(func, closures):
try:
cell_contents = c.cell_contents
except ValueError: # cell is empty
continue
yield name, c.cell_contents
return dict(get_cell_contents())
# thanks to Davies Liu for recursion of globals
def nestedglobals(func, recurse=True):
"""get the names of any globals found within func"""
func = code(func)
if func is None: return list()
import sys
from .temp import capture
CAN_NULL = sys.hexversion >= 0x30b00a7 # NULL may be prepended >= 3.11a7
names = set()
with capture('stdout') as out:
dis.dis(func) #XXX: dis.dis(None) disassembles last traceback
for line in out.getvalue().splitlines():
if '_GLOBAL' in line:
name = line.split('(')[-1].split(')')[0]
if CAN_NULL:
names.add(name.replace('NULL + ', ''))
else:
names.add(name)
for co in getattr(func, 'co_consts', tuple()):
if co and recurse and iscode(co):
names.update(nestedglobals(co, recurse=True))
return list(names)
def referredglobals(func, recurse=True, builtin=False):
"""get the names of objects in the global scope referred to by func"""
return globalvars(func, recurse, builtin).keys()
def globalvars(func, recurse=True, builtin=False):
"""get objects defined in global scope that are referred to by func
return a dict of {name:object}"""
if ismethod(func): func = func.__func__
if isfunction(func):
globs = vars(getmodule(sum)).copy() if builtin else {}
# get references from within closure
orig_func, func = func, set()
for obj in orig_func.__closure__ or {}:
try:
cell_contents = obj.cell_contents
except ValueError: # cell is empty
pass
else:
_vars = globalvars(cell_contents, recurse, builtin) or {}
func.update(_vars) #XXX: (above) be wary of infinte recursion?
globs.update(_vars)
# get globals
globs.update(orig_func.__globals__ or {})
# get names of references
if not recurse:
func.update(orig_func.__code__.co_names)
else:
func.update(nestedglobals(orig_func.__code__))
# find globals for all entries of func
for key in func.copy(): #XXX: unnecessary...?
nested_func = globs.get(key)
if nested_func is orig_func:
#func.remove(key) if key in func else None
continue #XXX: globalvars(func, False)?
func.update(globalvars(nested_func, True, builtin))
elif iscode(func):
globs = vars(getmodule(sum)).copy() if builtin else {}
#globs.update(globals())
if not recurse:
func = func.co_names # get names
else:
orig_func = func.co_name # to stop infinite recursion
func = set(nestedglobals(func))
# find globals for all entries of func
for key in func.copy(): #XXX: unnecessary...?
if key is orig_func:
#func.remove(key) if key in func else None
continue #XXX: globalvars(func, False)?
nested_func = globs.get(key)
func.update(globalvars(nested_func, True, builtin))
else:
return {}
#NOTE: if name not in __globals__, then we skip it...
return dict((name,globs[name]) for name in func if name in globs)
def varnames(func):
"""get names of variables defined by func
returns a tuple (local vars, local vars referrenced by nested functions)"""
func = code(func)
if not iscode(func):
return () #XXX: better ((),())? or None?
return func.co_varnames, func.co_cellvars
def baditems(obj, exact=False, safe=False): #XXX: obj=globals() ?
"""get items in object that fail to pickle"""
if not hasattr(obj,'__iter__'): # is not iterable
return [j for j in (badobjects(obj,0,exact,safe),) if j is not None]
obj = obj.values() if getattr(obj,'values',None) else obj
_obj = [] # can't use a set, as items may be unhashable
[_obj.append(badobjects(i,0,exact,safe)) for i in obj if i not in _obj]
return [j for j in _obj if j is not None]
def badobjects(obj, depth=0, exact=False, safe=False):
"""get objects that fail to pickle"""
from dill import pickles
if not depth:
if pickles(obj,exact,safe): return None
return obj
return dict(((attr, badobjects(getattr(obj,attr),depth-1,exact,safe)) \
for attr in dir(obj) if not pickles(getattr(obj,attr),exact,safe)))
def badtypes(obj, depth=0, exact=False, safe=False):
"""get types for objects that fail to pickle"""
from dill import pickles
if not depth:
if pickles(obj,exact,safe): return None
return type(obj)
return dict(((attr, badtypes(getattr(obj,attr),depth-1,exact,safe)) \
for attr in dir(obj) if not pickles(getattr(obj,attr),exact,safe)))
def errors(obj, depth=0, exact=False, safe=False):
"""get errors for objects that fail to pickle"""
from dill import pickles, copy
if not depth:
try:
pik = copy(obj)
if exact:
assert pik == obj, \
"Unpickling produces %s instead of %s" % (pik,obj)
assert type(pik) == type(obj), \
"Unpickling produces %s instead of %s" % (type(pik),type(obj))
return None
except Exception:
import sys
return sys.exc_info()[1]
_dict = {}
for attr in dir(obj):
try:
_attr = getattr(obj,attr)
except Exception:
import sys
_dict[attr] = sys.exc_info()[1]
continue
if not pickles(_attr,exact,safe):
_dict[attr] = errors(_attr,depth-1,exact,safe)
return _dict
# EOF
|