# HG changeset patch # Parent 5a05c0eeefc3c9d05aa55b6e5fa2c272cd22d619 Issue #26439: ctypes.util.find_library() support on AIX diff -r 5a05c0eeefc3 Lib/ctypes/__init__.py --- a/Lib/ctypes/__init__.py Sat Aug 27 04:07:54 2016 +0000 +++ b/Lib/ctypes/__init__.py Sat Oct 01 06:26:31 2016 +0000 @@ -337,6 +337,16 @@ flags |= _FUNCFLAG_USE_ERRNO if use_last_error: flags |= _FUNCFLAG_USE_LASTERROR + if _sys.platform.startswith("aix"): + """ + When the name contains ".a(" and ends with ")", as in "libFOO.a(libFOO.so)", + this is taken to be an archive(member) syntax for dlopen(), and the mode is adjusted + Otherwise, name is presented to dlopen() as a file argument + """ + if name and name.endswith(")") and ".a(" in name: + # from _ctypes import RTLD_MEMBER + RTLD_MEMBER = 0x00040000 + mode |= RTLD_MEMBER class _FuncPtr(_CFuncPtr): _flags_ = flags diff -r 5a05c0eeefc3 Lib/ctypes/_aix.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/ctypes/_aix.py Sat Oct 01 06:26:31 2016 +0000 @@ -0,0 +1,277 @@ +""" +Lib/ctypes support for LoadLibrary interface to dlopen() for AIX +Similar kind of support (i.e., as a separate file) +as has been done for Darwin support ctypes.macholib.* +rather than as separate, detailed if: sections in utils.py + +dlopen() is an interface to AIX initAndLoad() - primary documentation at: +https://www.ibm.com/support/knowledgecenter/ssw_aix_53/com.ibm.aix.basetechref/doc/basetrf1/dlopen.htm?lang=en +https://www.ibm.com/support/knowledgecenter/ssw_aix_53/com.ibm.aix.basetechref/doc/basetrf1/load.htm?lang=en +""" +# Author: M Felt, aixtools.net, Feb-Sept 2016 +# I thank Martin Panter for his patience and comments + +import re, os, sys +from . import util + +def aixABI(): + """ + Internal support function: + return executable size - 32-bit, or 64-bit + This is vital to the search for suitable member in an archive + """ + if sys.maxsize < 2**32: + return 32 + else: + return 64 + +def get_dumpH(file): + """ + Internal support function: + dump -H output provides info on archive/executable contents + and related paths. This function call dump -H as a subprocess + and returns a list of (object, objectinfo) tuples. + Note: shell=False, i.e., /usr/bin/dump is called without shell assistance + """ + #dumpH parsing: + # 1. Line starts with /, ./, or ../ - set object name + # 2. If "INDEX" return object + # 3. get extra info (lines starting with [0-9]) + + import subprocess + def get_object(p): + object = None + for x in p.stdout: + if x.startswith(('/', './', '../')): + object = x + elif "INDEX" in x: + return object.rstrip('\n') + return None + + def get_objectinfo(p): + # as an object was found, return known paths, archives and members + # these lines start with a digit + lines = [] + for line in p.stdout: + if re.match("[0-9]", line): + lines.append(line) + else: + # Should be a blank separator line, safe to consume + break + return lines + + p = subprocess.Popen(["/usr/bin/dump", "-X%s" % aixABI(), "-H", file], + universal_newlines=True, stdout=subprocess.PIPE) + + objects = [] + while True: + object = get_object(p) + if object is None: + break + objects.append((object, get_objectinfo(p))) + + p.stdout.close() + p.wait() + return objects + +def get_shared(input): + """ + Internal support function: + examine the get_dumpH() output and return a list of all shareable objects + indicated in the output + the character "[" is used to strip off the path information + Note: the "[" and "]" characters that are part of dump -H output are not processed here + """ + list = [] + for (line, _) in input: + # potential member lines contain "[" + # otherwise, no processing needed + if "[" in line: + # Strip off trailing colon (:) + list.append(line[line.index("["):-1]) + return list + +def get_exactMatch(expr, lines): + """ + Internal support function: + Must be only one match, otherwise result is None + When there is a match, strip the leading "[" and trailing "]" + """ + # member names in the dumpH output are between square brackets + expr = r'\[(%s)\]' % expr + matches = list(filter(None, (re.search(expr, line) for line in lines))) + if len(matches) == 1: + return matches[0].group(1) + else: + return None + +# additional processing to deal with AIX legacy names for 64-bit members +def get_legacy(members): + """ + Internal support function: + This routine resolves historical aka legacy naming schemes started in AIX4 + for shared library support for library members names for, e.g., libc.a + This routine addresses specific issues for 64-bit legacy issues + """ + if aixABI() == 64: + # AIX 64-bit member is one of shr64.o, shr_64.o, or shr4_64.o + expr = r'shr4?_?64\.o' + member = get_exactMatch(expr, members) + if member: + return member + # 32-bit legacy names - shr.o and shr4.o -- shr.o is the default name since AIX5 + # so we look for shr.o first, i.e., shr4.o is returned only when shr.o does not exist + else: + for name in ['shr.o', 'shr4.o']: + member = get_exactMatch(re.escape(name), members) + if member: + return member + return None + +# In a new ABI (perhaps for Python3.7) could be called +# as a request for the latest version installed, or a specific version +def get_version(name, members): + """ + Internal support function: + examine member list and return highest numbered version - if it exists + + This function is called when an unversioned libFOO.a(libFOO.so) has not + been found + + Versioning for the member name is expected to follow GNU LIBTOOL conventions: + d.1 find [libFoo.so.XX] + d.2 find [libFoo.so.XX.YY] + d.3 find [libFoo.so.XX.Y.ZZZ] + + Unfortunately, before the GNU convention took hold there was a convention in AIX to + use GNU convention "as-is" for 32-bit archive members but in 64-bit members + add either 64 or _64 between libFOO and .so - so generally libFOO_64.so, but occasionally + (read notably for libssl.a and libcrypto.a) the member names are libFOO64.so + Generally, the second expression that looks for "64" in the member name will not be needed. + """ + # the expression ending for versions must start as + # '.so.[0-9]', i.e., *.so.[at least one digit] + # while multiple, more specific expressions could be specified + # to search for .so.X, .so.X.Y and .so.X.Y.Z + # we are accepting any combination of digits and '.' + # after the first required digit + # imho, anything other that .so.digits.digits.digits + # with only the first .digits required is someone purposely + # trying to break the parser + exprs = [r'lib%s\.so\.[0-9]+[0-9.]*' % name, + r'lib%s_?64\.so\.[0-9]+[0-9.]*' % name] + for expr in exprs: + versions = [] + for line in members: + m = re.search(expr, line) + if m: + versions.append(m.group(0)) + if versions: + return util._last_version(versions, '.') + return None + +def get_member(name, members): + """ + Internal support function: + Return an archive member matching name + Given a list of members find and return the most appropriate result + Priority is given to generic libXXX.so, then a versioned libXXX.so.a.b.c + and lastly, legacy AIX naming scheme + """ + # look first for a generic match - prepend lib and append .so + expr = r'lib%s\.so' % name + member = get_exactMatch(expr, members) + if member: + return member + # because an exact match with .so as extension of member name + # was not found, look for a versioned name + # When a versioned name is also not found, look for AIX legacy member name + member = get_version(name, members) + if member: + return member + else: + return get_legacy(members) + +def getExecLibPath_aix(): + """ + Internal support function: + On AIX, the buildtime searchpath is stored in the executable. the command dump -H can extract this info + Prefix searched libraries with LIBPATH, or LD_LIBRARY_PATH if defined + Additional paths are appended based on paths to libraries the python executable is linked with + This mimics AIX dlopen() behavior + """ + + libpaths = os.environ.get("LIBPATH") + if libpaths is None: + libpaths = os.environ.get("LD_LIBRARY_PATH") + if libpaths is None: + libpaths = [] + else: + libpaths = libpaths.split(":") + objects = get_dumpH(sys.executable) + for (_, lines) in objects: + for line in lines: + # the second (optional) argument is PATH if it includes a / + path = line.split()[1] + if "/" in path: + libpaths.extend(path.split(":")) + return libpaths + +def find_library(name): + """ + AIX specific routine - that knows how to find members in archives + + Find a library looking first for an archive (.a) with a suitable member, + return .so if found, otherwise highest version number + If no archive is found, look for a .so file (no versioning, symbolic link + must point to correct version) + """ + + def find_shared(paths, name): + """ + Search "paths" for archive, and if an archive is found + return the result of get_member(). + If no archives are found return None + paths is where to look for archives of the given name + name is the library name given to find_library() + """ + for dir in paths: + # /lib is a symbolic link to /usr/lib, skip it + if dir == "/lib": + continue + # "lib" is prefixed to emulate compiler name resolution, e.g., -lc to libc + base = 'lib%s.a' % name + archive = os.path.join(dir, base) + if os.path.exists(archive): + members = get_shared(get_dumpH(archive)) + member = get_member(re.escape(name), members) + if member != None: + return (base, member) + else: + return (None, None) + return (None, None) + + libpaths = getExecLibPath_aix() + (base, member) = find_shared(libpaths, name) + if base != None: + return "%s(%s)" % (base, member) + + # To get here, a member in an archive has not been found + # In other words, either: + # a) a .a file was not found + # b) a .a file did not have a suitable member + # So, look for a .so file + # Check libpaths for .so file + # Note, the installation must prepare a link from a .so + # to a versioned file + # This is common practice by GNU libtool on other platforms + soname = "lib%s.so" % name + for dir in libpaths: + # /lib is a symbolic link to /usr/lib, skip it + if dir == "/lib": + continue + shlib = os.path.join(dir, soname) + if os.path.exists(shlib): + return soname + # if we are here, we have not found anything plausible + return None diff -r 5a05c0eeefc3 Lib/ctypes/test/test_loading.py --- a/Lib/ctypes/test/test_loading.py Sat Aug 27 04:07:54 2016 +0000 +++ b/Lib/ctypes/test/test_loading.py Sat Oct 01 06:26:31 2016 +0000 @@ -15,6 +15,11 @@ libc_name = "coredll" elif sys.platform == "cygwin": libc_name = "cygwin1.dll" + elif sys.platform.startswith("aix"): + if sys.maxsize < 2**32: + libc_name = "libc.a(shr.o)" + else: + libc_name = "libc.a(shr_64.o)" else: libc_name = find_library("c") diff -r 5a05c0eeefc3 Lib/ctypes/util.py --- a/Lib/ctypes/util.py Sat Aug 27 04:07:54 2016 +0000 +++ b/Lib/ctypes/util.py Sat Oct 01 06:26:31 2016 +0000 @@ -3,6 +3,20 @@ import subprocess import sys +def _last_version(libnames, sep): + def _num_version(libname): + # "libxyz.so.MAJOR.MINOR" => [MAJOR, MINOR] + parts = libname.split(sep) + nums = [] + try: + while parts: + nums.insert(0, int(parts.pop())) + except ValueError: + pass + return nums or [sys.maxsize] + + return max(reversed(libnames), key=_num_version) + # find_library(name) returns the pathname of a library, or None. if os.name == "nt": @@ -90,6 +104,15 @@ continue return None +elif sys.platform.startswith("aix"): + # using a new file to not clutter up this one + # Behavior is to look for a member in an archive + # otherwise try to find a .so file + # uses info from /usr/bin/dump -H - rather than ldconfig -p + import ctypes._aix as aix + def find_library(name): + return aix.find_library(name) + elif os.name == "posix": # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump import re, tempfile @@ -180,17 +203,6 @@ if sys.platform.startswith(("freebsd", "openbsd", "dragonfly")): - def _num_version(libname): - # "libxyz.so.MAJOR.MINOR" => [ MAJOR, MINOR ] - parts = libname.split(b".") - nums = [] - try: - while parts: - nums.insert(0, int(parts.pop())) - except ValueError: - pass - return nums or [sys.maxsize] - def find_library(name): ename = re.escape(name) expr = r':-l%s\.\S+ => \S*/(lib%s\.\S+)' % (ename, ename) @@ -209,8 +221,7 @@ res = re.findall(expr, data) if not res: return _get_soname(_findLib_gcc(name)) - res.sort(key=_num_version) - return os.fsdecode(res[-1]) + return os.fsdecode(_last_version(res, b".")) elif sys.platform == "sunos5": @@ -323,10 +334,11 @@ print(find_library("msvcrt")) if os.name == "posix": - # find and load_version - print(find_library("m")) - print(find_library("c")) - print(find_library("bz2")) + # find + print("m\t:: %s" % find_library("m")) + print("c\t:: %s" % find_library("c")) + print("bz2\t:: %s" % find_library("bz2")) + print("crypt\t:: %s" % find_library("crypt")) # getattr ## print cdll.m @@ -338,6 +350,12 @@ print(cdll.LoadLibrary("libcrypto.dylib")) print(cdll.LoadLibrary("libSystem.dylib")) print(cdll.LoadLibrary("System.framework/System")) + elif sys.platform.startswith("aix"): + print("crypto\t:: %s" % cdll.LoadLibrary(find_library("crypto"))) + if sys.maxsize < 2**32: + print("c\t:: %s" % cdll.LoadLibrary("libc.a(shr.o)")) + else: + print("c\t:: %s" % cdll.LoadLibrary("libc.a(shr_64.o)")) else: print(cdll.LoadLibrary("libm.so")) print(cdll.LoadLibrary("libcrypt.so"))