# HG changeset patch # Parent 6d1b6a68f775fada9877d295e62958bafa1ca11e diff -r 6d1b6a68f775 Lib/ctypes/__init__.py --- a/Lib/ctypes/__init__.py Sat Dec 05 11:45:17 2015 -0800 +++ b/Lib/ctypes/__init__.py Mon Jun 20 03:34:09 2016 +0000 @@ -355,6 +355,18 @@ flags |= _FUNCFLAG_USE_ERRNO if use_last_error: flags |= _FUNCFLAG_USE_LASTERROR + if _sys.platform.startswith("aix"): + # adding RTLD_NOW is already 'forced' in Modules/_ctypes/callproc.c + # left here (as a comment) to remind it is required by AIX + # from _ctypes import RTLD_NOW - not until Python3.6 + # mode |= RTLD_NOW -- not documented, but added automatically by Modules/_ctypes/_ctypes.c + # When a string contains ".a(" and ends with ")", asin "libFOO.a(libFOO.so)" + # this is taken to be an archive(member) syntax for AIX, and the mode is adjusted + + if name and name.endswith(")") and name.rfind(".a(") > 0: + # from _ctypes import RTLD_MEMBER is not expected to be valid before Python3.6 + RTLD_MEMBER = 0x00040000 + mode |= RTLD_MEMBER class _FuncPtr(_CFuncPtr): _flags_ = flags diff -r 6d1b6a68f775 Lib/ctypes/_aixutil.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/ctypes/_aixutil.py Mon Jun 20 03:34:09 2016 +0000 @@ -0,0 +1,303 @@ +# Lib/ctype support for cdll interface to dlopen() for AIX +# Author: M Felt, aixtools.net, May 2016 +# +# dlopen() is an interface to AIX initAndLoad() - these are documented at: +# https://www.ibm.com/support/knowledgecenter/ssw_aix_53/com.ibm.aix.basetechref/doc/basetrf1/dlopen.htm?lang=en +# https://www.ibm.com/support/knowledgecenter/ssw_aix_53/com.ibm.aix.basetechref/doc/basetrf1/load.htm?lang=en +# +# Some basics +# when given an argument such as -lFOO to the AIX loader and a symbol must be resolved +# the AIX ld searches LIBPATH for an archive named libFOO.a. When found, it then searches archive members +# for a member that resolves the symbol. +# When an archive (.a) is not found, the search starts again, but looking for a file named libFOO.so +# +# when a full or partial path information is given dlopen() does not search LIBPATH - only the path specified +# +# ctypes._aixutil.find_library() searches archives for members +# if a member is not found it looks for "name.so" in the same directory paths +# +# find_library("libFOO.so") is not an intended usage (looks for libFOO.so.a(libFOO.so.so) or libFOO.so.so) +# That may be suitable for CDLL(), but not for find_library() +# Much code is written as: CDLL(find_library("libFOO")) +# or : CDLL((find_library("FOO")) +# +# examples: +# aix.find_library("libiconv") => libiconv.a(shr4_64.o) || libiconv.a(shr4.o) (64 or 32 bit modes) +# aix.find_library("libintl") => libintl.a(libintl.so.1) or libintl.a(libintl.so.8) depending on the +# version of GNU gettext installed + +# when packaging python - a packager (for AIX) may want to include "export LDFLAGS=-L${prefix}/lib" +# so that dlopen() will look in ${prefix}/lib as well /usr/lib, etc.. + +# +# NOTE: RTLD_NOW is AIX default, regardless (as of March 2016) (set above!) +# NOTE: RTLD_MEMBER is needed by dlopen to open a shared object from within an archive +# +# find_library() returns the archive(member) string needed by dlopen() to open a member whenever possible +# only returning a file - when available - if the archive search fails +# directory path is returned only when directory path information was supplied +# +# Note: in the functions below local parameters might be declared to ensure scope is local + +import re, os, sys + +# executable "size" is important +# archives can contain both 32 and 64-bit sizes - with different names +# e.g., shr.o (32-bit), shr_64.o (64-bit) +# Note: the names may be equal, e.g., libssl.so +# shared objects may have different search paths (e.g., /usr/lib (32-bit), /usr/lib64 (64-bit)) +# especially when they have dependancies on files rather than archives +def aixABI(): + if (sys.maxsize < 2**32): + return 32 + else: + return 64 + +# return striped output of /usr/bin/dump ... -H ... +def get_dumpH(object, p=None, lines=None): + import subprocess + p = subprocess.Popen(["/usr/bin/dump", "-X%s"%aixABI(), "-H", object], + universal_newlines=True, stdout=subprocess.PIPE) + # not interested in blank lines, leading/trailing spaces + lines=[line.strip() for line in p.stdout.readlines() if line.strip()] + # close process and wait (verify) it has completed + p.stdout.close() + p.wait() + # print lines + return lines + +# process dumpH output +# return list of members with loader sections, i.e., shareable +# An excerpt of (stripped) dump -X64 -H output looks like: +# Note: a third # is part of the dump output! + +##/usr/lib/libc.a[vsaveres_64.o]: +##Loader section is not available +##/usr/lib/libc.a[ptrgl_64_64.o]: +##Loader section is not available +##/usr/lib/libc.a[shr_64.o]: +##***Loader Section*** +##Loader Header Information +##VERSION# #SYMtableENT #RELOCent LENidSTR +##0x00000001 0x00000ab4 0x00002c63 0x0000002d +###IMPfilID OFFidSTR LENstrTBL OFFstrTBL +##0x00000003 0x0003c748 0x00009121 0x0003c775 +##***Import File Strings*** +##INDEX PATH BASE MEMBER +##0 /usr/lib:/lib +##1 / unix +##2 libcrypt.a shr_64.o +##/usr/lib/libc.a[posix_aio_64.o]: +##***Loader Section*** +##Loader Header Information + +def get_shared(input, list=None, cpy=None, idx=0): + cpy=input + list=[] + for line in input: + idx = idx + 1 + # if line does not start with "/" - it is superfluous + if line.startswith("/"): + next = cpy.pop(idx) + # if next line does not contain the word "not", then it has loader information + # and the member including leftmost "[" and rightmost "]" needs to be put on the list + # e.g., [shr_64.o] from: "/usr/lib/libc.a[shr_64.o]:" + if not "not" in next: + list.append(line[line.find("["):line.rfind(":")]) + return(list) + +# looking for ONE exact match, otherwise return None +def get_match(expr, lines, line=None, member=None): + matches = [m.group(0) for line in lines for m in [re.search(expr, line)] if m] + if len(matches) == 1: + match=matches[0] + # extract member name between leftmost "[" and rightmost "]" and return + member = match[match.find("[")+1:match.find("]")] + return member + else: + return None + +# sometimes a generic name gets 64 appended to it, and AIX 64-bit +# archive members have different legacy names than 32-bit ones +# the leading and trailing "[" and "]", respectively, are part of the +# search string to distinquish between archive (or BASE) and MEMBER +# see "INDEX PATH BASE MEMBER" header informatin above for an explanation of positions +def get_member64(name, members): + # This routine is only called when a libFOO.so was not found earlier + # CHECKME: change logic to do that search here as well? + # Recall that member names are in square brackets [] + # an old convention - insert _64 between libFOO and .so + # Note: assumption - lib has been prepended (if needed) + # and .so is not part of the name + # '\[%s_*64\.so\]' % name, -> has either _64 or 64 added to name + # Note: "version" line succeeds here only when there is only one versioned member. + # '\[%s.so.[0-9]+.*\]' % name, -> needs versioning (when no version was specified) + # '\[%s_*64\.o\]' % name, -> legacy name AIX scheme for libFOO 64-bit .so + # '\[shr4*_*64.o\]']: -> legacy AIX names: shr_64.o, shr_64.o, shr64.o + for expr in [ + '\[%s_*64\.so\]' % name, + '\[%s.so.[0-9]+.*\]' % name, + '\[%s_*64\.o\]' % name, + '\[shr4*_*64.o\]']: + member = get_match(expr, members) + if member: + return member + return None + +# Get the most recent/highest numbered version - if it exists +# Only called if a versioned is requested - CHECK - superfluous due to find_library() convention? +def get_version(name, members): + def _num_version(libname): + # "libxyz.so.CURRENT.REVISION.AGE" => [ CURRENT, REVISION, AGE ] + parts = libname.split(".") + nums = [] + try: + while parts: + nums.insert(0, int(parts.pop())) + except ValueError: + pass + return nums or [ sys.maxsize ] + + expr = '%s[_64]*.so.[0-9]+[0-9\.]*' % name + versions = [m.group(0) for line in members for m in [re.search(expr, line)] if m] + if versions: + versions.sort(key=_num_version) + return versions[-1] + return None + +# return an archive member matching name +# (versioned) .so members have priority over legacy AIX member name +# +# member names in in the dumpH output are in square brackets +# These get stripped by get_match() and regular parenthesis are added by the caller +def get_member(name, members, member=None): + # look first for a generic match + # '\[%s\]' % name, -> CONCEIVEABLE: specific member requested + # '\[%s\.so\]' % name -> most common, append .so to name + for expr in [ + '\[%s\.so\]' % name]: + member = get_match(expr, members) + if member: + return member + if not member: + member = get_version(name, members) + if member: + return member + elif aixABI() == 64: + return(get_member64(name, members)) + # 32-bit legacy names - shr4.o and shr.o + else: + for expr in [ + '\[shr4.o\]', + '\[shr.o\]']: + member = get_match(expr, members) + if member: + return member + return None + +def getExecLibPath_aix(libpaths=None, x=None, lines=None, line=None, skipping=None): + # if LIBPATH is defined, add it to the paths to search + # FIXME: maybe LD_LIBRARY_PATH should be checked as well + libpaths = os.environ.get("LIBPATH") + lines = get_dumpH(sys.executable) + skipping = True; + for line in lines: + if skipping == False: + x = line.split()[1] + if (x.startswith('/') or x.startswith('./') or x.startswith('../')): + if libpaths is not None: + libpaths = "%s:%s" % (libpaths, x) + else: + libpaths = x + elif line.find("INDEX PATH") == 0: + skipping = False; + return libpaths + +# Considerations: +# A) is there PATH information (if so, check only that directory), else determine paths +# B) if name ends with .so.digit(s) - a specific version is requested +## if so, look for "generic" archive name + specific member name +# option C - common requests +# C.1) FOO -> libFOO.a(libFOO.so) +# C.2) FOO -> libFOO.a(libFOO.so.X.Y.Z - latest version) +# C.3) FOO -> libFOO.a((AIX legacy name)) + +##### A) if an argument resolves as a path.filename, use as path part as "paths" +def find_parts(name): + pathe = name.rfind("/") + if pathe > 0: + parts = (name[0:pathe], name[pathe+1:]) + paths = parts[0] + name = parts[1] + else: + paths = getExecLibPath_aix() + if name.find("lib") != 0: + name = "lib%s" % name + return (paths, name) + +def find_library(name, _name=None, lines=None, path=None, member=None, shlib=None): + parts = find_parts(name) + paths = parts[0] + _name = parts[1] + + ##### B) the default AIX behavior is to find a member within a .a "base" archive + ## FIXME: if name endswith(".so.\D+" then: + ## we are looking for a specific version, e.g., libssl.so.0.9.8 => libssl.a(libssl.so.0.9.8) + _base = _name.rsplit(".so")[0] + for dir in paths.split(":"): + # /lib is a symbolic link to /usr/lib, skip it + if dir == "/lib": + continue + basefile = os.path.join(dir, "%s.a" % _base) + if os.path.exists(basefile): + members = get_shared(get_dumpH(basefile)) + member = get_member(_name, members) + if member != None: + if name.rfind("/") > 0: + # PATH is significant + shlib = "%s/%s.a(%s)" % (parts[0], _base, member) + else: + # SEARCHing is significant + shlib = "%s.a(%s)" % (_base, member) + return shlib + ##### C) look for a FILE based on the request + # Assumptions: + # Since it is not in an archive - considered a special case + # when .so is in filename + # if .so is not in name, append .so to filename before testing + # If path information is included in "name" return the path, + # or the path.so if either exists + if name.rfind("/") > 0: + if os.path.exists(name): + return(name) + else: + # add .so to name, just in case + shlib = "%s.so" % name + if os.path.exists(name): + return(shlib) + # did not find it based on literal "name", so try extra possibilities + # based on _name + # if .so is specified to find_library() then something specific is wanted + # use the name as supplied + if _name.find(".so") > 0: + _name = name.rsplit("/")[-1] + exact = 1 + else: + exact = 0 + for dir in paths.split(":"): + # /lib is a symbolic link to /usr/lib, skip it + if dir == "/lib": + continue + if exact == 1: + shlib = os.path.join(dir, "%s" % _name) + else: + shlib = os.path.join(dir, "%s.so" % _base) + if os.path.exists(shlib): + if name.rfind("/") > 0: + # PATH is significant + return shlib + else: + # SEARCHing is significant + return shlib.rsplit("/")[-1] + # if we are here, we have not found anything plausible + return None diff -r 6d1b6a68f775 Lib/ctypes/test/test_loading.py --- a/Lib/ctypes/test/test_loading.py Sat Dec 05 11:45:17 2015 -0800 +++ b/Lib/ctypes/test/test_loading.py Mon Jun 20 03:34:09 2016 +0000 @@ -27,6 +27,16 @@ CDLL(os.path.basename(libc_name)) self.assertRaises(OSError, CDLL, self.unknowndll) + @unittest.skipUnless(sys.platform.startswith ("aix"), + 'test specific to AIX') + def test_load_AIX_libc(self): + if (sys.maxsize < 2**32): + cdll.LoadLibrary("libc.a(shr.o)") + else: + cdll.LoadLibrary("libc.a(shr_64.o)") + self.assertRaises(OSError, cdll.LoadLibrary, "libc.so.6") + self.assertRaises(OSError, CDLL, self.unknowndll) + @unittest.skipUnless(libc_name is not None, 'could not find libc') @unittest.skipUnless(libc_name is not None and os.path.basename(libc_name) == "libc.so.6", @@ -38,7 +48,7 @@ self.assertRaises(OSError, cdll.LoadLibrary, self.unknowndll) def test_find(self): - for name in ("c", "m"): + for name in ("c", "m", "intl", "ssl"): lib = find_library(name) if lib: cdll.LoadLibrary(lib) diff -r 6d1b6a68f775 Lib/ctypes/util.py --- a/Lib/ctypes/util.py Sat Dec 05 11:45:17 2015 -0800 +++ b/Lib/ctypes/util.py Mon Jun 20 03:34:09 2016 +0000 @@ -71,7 +71,15 @@ def find_library(name): return name -if os.name == "posix" and sys.platform == "darwin": +if sys.platform.startswith("aix"): + # find .so members in .a files + # using dump loader header information + sys. + import ctypes._aixutil as aix + + def find_library(name): + return aix.find_library(name) + +elif os.name == "posix" and sys.platform == "darwin": from ctypes.macholib.dyld import dyld_find as _dyld_find def find_library(name): possible = ['lib%s.dylib' % name, @@ -256,6 +264,21 @@ print find_library("m") print find_library("c") print find_library("bz2") + if sys.platform.startswith("aix"): + # examples of working with AIX and versions, when available + print "find_library(\"c\") returns:", find_library("c") + print "find_library(\"libc\") returns:", find_library("libc") + print "find_library(\"libssl\") returns:", find_library("libssl") + # expectation is None:archive libssl64.a with member libssl64.so is non-standard + print "find_library(\"libssl64\") returns:", find_library("libssl64") + print "find_library(\"ssl\") returns:", find_library("ssl") + print "find_library(\"libiconv\") returns:", find_library("libiconv") + print "find_library(\"intl\") returns:", find_library("intl") + print "sudoers.so: ",(find_library("sudoers.so")) + print "sudoers.so: ",(cdll.LoadLibrary("sudoers.so")) + print "sudoers.so: ",(find_library("/opt/libexec/sudo/sudoers.so")) + print "sudo_util: ",(find_library("/opt/libexec/sudo/sudo_util")) + print "sudo_util: ",(find_library("sudo_util")) # getattr ## print cdll.m @@ -268,9 +291,16 @@ print cdll.LoadLibrary("libSystem.dylib") print cdll.LoadLibrary("System.framework/System") else: - print cdll.LoadLibrary("libm.so") - print cdll.LoadLibrary("libcrypt.so") print find_library("crypt") + if sys.platform.startswith("aix"): + # AIX returns different values for 32 and 64-bit looking for find_library("c"), etc. + print(cdll.LoadLibrary(find_library("c"))) + print(cdll.LoadLibrary(find_library("/usr/lib/libc"))) + print(cdll.LoadLibrary(find_library("/opt/libexec/sudo/sudoers.so"))) + print cdll.LoadLibrary(find_library("crypt")) + else: + print(cdll.LoadLibrary("libm.so")) + print cdll.LoadLibrary("libcrypt.so") if __name__ == "__main__": test()