# Author: M Felt, aixtools.net, May 2016 # ctype support for cdll interface to dlopen() for AIX # find .so aka shared library members in .a files (i.e., archives) # # dlopen() is an interface to AIX initAndLoad() - these are documented at: # https://www.ibm.com/support/knowledgecenter/ssw_aix_53/com.ibm.aix.basetechref/doc/basetrf1/dlopen.htm?lang=en # https://www.ibm.com/support/knowledgecenter/ssw_aix_53/com.ibm.aix.basetechref/doc/basetrf1/load.htm?lang=en # # like GNU, when a full-path is given dlopen() does not search for the shared library # but just opens the file. Until this addition, that was the only real hope AFTER the # files were ALSO installed in places like /usr/lib and /usr/lib64 (cdll would find these) # as well as a full or relative path name. # Frequent issues with AIX and ctypes.util.find_library() is that the "posix" code, # by default, relied on the existance of the output of "ldconfig -p" to locate shared library locations # This caused delays, as it is unusual for ldonfig to be available, and unknown if # if could report AIX legacy shared members that /bin/rtld would find in normal operations # # Think of ctypes.aix.find_library() as the utility that searches .a archives for the member you specify # this can be as either a stem - as common with linking - "-lc" or find_library("c") # also accepted is the stem prefixed by "lib", e.g., find_library("libintl") # When prefixed by "lib" a suffix (is common practice) of ".so" is accepted, e.g. find_library("libc.so") # Finally, also common in python code is a versioned specification, e.g. find_library("libintl.so.1") # in these cases, a member is found only if the versions match exactly # # examples: # aix.find_library("libiconv.so") libiconv.a(shr4_64.o) # aix.find_library("libintl.so") libintl.a(libintl.so.1) # aix.find_library("libintl.so.1") libintl.a(libintl.so.1) # aix.find_library("libintl.so.2") None ## should be None! # aix.find_library("libintl.so.8") None ## might be: libintl.a(libintl.so.8) e.g., if aixtools.gnu.gettext installed # Currently, my packaging of python does not insert ${prefix}/lib in the library search path # instead, if LIBPATH is not already defined, it is defined in the environment so that # dlopen() will look there as well. # As python executable probably does not include the $prefix/lib # implied in the ./configure --prefix=/foo/dir # then LIBPATH, when not already defined, is set to sys.prefix/lib # # With this 'assist' AIX dlopen() will also search ${prefix}/lib for libraries # # NOTE: RTLD_NOW is AIX default, regardless (as of March 2016) (set above!) # NOTE: RTLD_MEMBER is needed by dlopen to open a shared object from within an archive # # find_library() returns the archive(member) string needed by dlopen() to open a member # as part of an archive, rather than finding a directory path - more in line with default dlopen() processing # # when a 'generic' .so file is requested, e.g., libintl.so, of libc.so # - if something is available, that is returned # # "stem" names, such as "c" or "intl" find the latest available version, # or nothing, if a shared library does not exist (e.g., "m", and "libm" will probably fail) import re as _re, os as _os, sys as _sys # need machine size to find the correct member in an archive def _aixABI(): if (_sys.maxsize < 2**32): return 32 else: return 64 # execute and pre-scan output of /usr/bin/dump ... -H ... def _get_dumpH(_exe_so_or_a, p=None, lines=None): import subprocess as _subprocess p = _subprocess.Popen(["/usr/bin/dump", "-X%s"%_aixABI(), "-H", _exe_so_or_a], universal_newlines=True, stdout=_subprocess.PIPE) lines=[line.strip() for line in p.stdout.readlines() if line.strip()] p.stdout.close() p.wait() return lines # process dumpH output and return only members with loader sections def _get_shared(input, list=None, cpy=None, idx=0): cpy=input list=[] for line in input: idx = idx + 1 if line.startswith("/"): next = cpy.pop(idx) if not "not" in next: list.append(line[line.find("["):line.rfind(":")]) return(list) # find the singular match, if more - undefined, and None is None def _get_match(expr, lines, line=None, member=None): matches = [m.group(0) for line in lines for m in [_re.search(expr, line)] if m] if len(matches) == 1: match=matches[0] member = match[match.find("[")+1:match.find("]")] return member else: return None # sometimes a generic name gets 64 appended to it, and AIX 64-bit # archive members have different legacy names than 32-bit ones def _get_member64(stem, members): # Recall that member names are in square brackets [] # an old convention - insert _64 between libFOO and .so for expr in [ '\[lib%s_*64\.so\]' % stem, '\[lib%s.so.[0-9]+.*\]' % stem, '\[lib%s_*64\.o\]' % stem, '\[shr4_64.o\]', '\[shr4*_*64.o\]']: member = _get_match(expr, members) if member: return member return None # Get the most recent/highest numbered version - if it exists def _get_version(stem, members): # sort function, borrowed from below def _num_version(libname): # "libxyz.so.MAJOR.MINOR" => [ MAJOR, MINOR ] parts = libname.split(".") nums = [] try: while parts: nums.insert(0, int(parts.pop())) except ValueError: pass return nums or [ _sys.maxsize ] expr = r'\[lib%s.so.[0-9]+.*]' % stem versions = [m.group(0) for line in members for m in [_re.search(expr, line)] if m] if versions: versions.sort(key=_num_version) return versions[-1][1:-1] return None # return an archive member matching name # (versioned) .so members have priority over legacy AIX member name # # member names in in the dumpH output are in square brackets # These get stripped by _get_match() and regular parenthesis are added by the caller def _get_member(stem, name, members): # look first for an exact match, then a generic match for expr in [ '\[%s\]' % name, '\[lib%s\.so\]' % stem]: member = _get_match(expr, members) if member: return member # if name ends in a digit, a specific version is wanted, # if not found above as an exact match do not look for # a different version if _re.findall("\d$", name): return None else: member = _get_version(stem, members) if member: return member elif _aixABI() == 64: return(_get_member64(stem, members)) # 32-bit legacy names - shr4.o and shr.o else: for expr in [ '\[shr4.o\]', '\[shr.o\]']: member = _get_match(expr, members) if member: return member return None # make an LIBPATH like string that will be searched directory by directory # where the first match is THE match def _getExecLibPath_aix(libpaths=None, x=None, lines=None, line=None, skipping=None): libpaths = _os.environ.get('LIBPATH') lines = _get_dumpH(_sys.executable) skipping = True; for line in lines: if skipping == False: x = line.split()[1] if (x.startswith('/') or x.startswith('./') or x.startswith('../')): if libpaths is not None: libpaths = "%s:%s" % (libpaths, x) else: libpaths = x elif line.find("INDEX PATH") == 0: skipping = False; return libpaths # A) exact match (relative/full-path name) # B) normal dlopen() processing, member in a archive # C) OLD behavior, "gnu"/"posix" like - find a file in a directory def find_library(name, _name=None, _stem=None, envpath=None, lines=None, path=None, member=None, shlib=None): ##### A) if an argument resolves as a filename, return ASIS # This is for people who are trying to match a file as "./libFOO.so" # or using a fixed (fullpath) if _os.path.exists(name): return name ##### B) the default AIX behavior is to find a member within a .a archive # find the shared library for AIX # name needs to in it's shortest form: libFOO.a, libFOO.so.1.0.1, etc. # are all FOO, as in link arguments -lFOO _stem = "%s." % name if _stem.find("lib") < 0: _stem = _stem[0:_stem.find(".")] else: _stem = _stem[3:_stem.find(".")] # for how LIBPATH and/or LD_LIBRARY_LD are used by dlopen() (actually loadAndInit()) # the expectation is that normal python packaging, on AIX, will not include ${prefix}/lib in # the library search path - aka -W,-blibpath value - we add it here when LIBPATH is not defined already # if the GNU autotools are configured to add ${prefix}/lib for all compilers, and also ${prefix)/lib64 # when using gcc (that may expect seperate directories rather than .a archives) envpath = _os.environ.get('LIBPATH') if envpath is None: _os.environ['LIBPATH'] = "%s/lib" % _sys.prefix paths = _getExecLibPath_aix() for dir in paths.split(":"): if dir == "/lib": continue # AIX rtl searches .a, by default, if it exists _arfile = _os.path.join(dir, "lib%s.a" % _stem) if _os.path.exists(_arfile): members = _get_shared(_get_dumpH(_arfile)) member = _get_member(_stem, name, members) if member != None: shlib = "lib%s.a(%s)" % (_stem, member) return shlib ##### C) look for a .so FILE based on the request if envpath is None: if _aixABI() == 64: _os.environ['LIBPATH'] = "%s/lib64" % _sys.prefix paths = _getExecLibPath_aix() # same as above! so comments only # else: # _os.environ['LIBPATH'] = "%s/lib" % _sys.prefix # paths = _getExecLibPath_aix() for dir in paths.split(":"): if dir == "/lib": continue shlib = _os.path.join(dir, "%s" % name) if _os.path.exists(shlib): return shlib # Try for a generic match shlib = _os.path.join(dir, "lib%s.so" % _stem) if _os.path.exists(shlib): return shlib # if we are here, we have not found anything plausible return None