Message325811
I've been trying to track down a leak in aiohttp: https://github.com/aio-libs/aiohttp/issues/3010
it seems like this leak now occurs with raw asyncio SSL sockets.
when the gist script is run like so: python3.7 `which mprof` run --interval=1 ~/dev/test_leak.py -test asyncio_test
it slowly leaks memory. This is effectively doing the following:
URLS = {
'https://s3.us-west-2.amazonaws.com/archpi.dabase.com/style.css': {
'method': 'get',
'headers': {'User-Agent': 'Botocore/1.8.21 Python/3.6.4 Darwin/17.5.0', 'X-Amz-Date': '20180518T025044Z', 'X-Amz-Content-SHA256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': f'AWS4-HMAC-SHA256 Credential={CREDENTIAL}/20180518/us-west-2/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=ae552641b9aa9a7a267fcb4e36960cd5863e55d91c9b45fd39b30fdcd2e81489', 'Accept-Encoding': 'identity'}
},
'https://s3.ap-southeast-1.amazonaws.com/archpi.dabase.com/doesnotexist': {
'method': 'GET' if sys.argv[1] == 'get_object' else 'HEAD',
'headers': {'User-Agent': 'Botocore/1.8.21 Python/3.6.4 Darwin/17.5.0', 'X-Amz-Date': '20180518T025221Z', 'X-Amz-Content-SHA256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': f'AWS4-HMAC-SHA256 Credential={CREDENTIAL}/20180518/ap-southeast-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=7a7675ef6d70cb647ed59e02d532ffa80d437fb03976d8246ea9ef102d118794', 'Accept-Encoding': 'identity'}
}
}
class HttpClient(asyncio.streams.FlowControlMixin):
transport = None
def __init__(self, *args, **kwargs):
self.__url = kwargs.pop('url')
self.__logger = logging.getLogger()
super().__init__()
def connection_made(self, transport):
self.transport = transport
url_parts = urlparse(self.__url)
entry = URLS[self.__url]
body = f'{entry["method"]} {url_parts.path} HTTP/1.1\r\nAccept: */*\r\nHost: {url_parts.hostname}\r\n'
for name, value in entry['headers'].items():
body += f'{name}: {value}\r\n'
body += '\r\n'
self.transport.write(body.encode('ascii'))
self.__logger.info(f'data sent: {body}')
def data_received(self, data):
self.__logger.info(f'data received: {data}')
self.transport.close()
# asyncio.get_event_loop().call_later(1.0, )
def eof_received(self):
self.__logger.info('eof_received')
def connection_lost(self, exc):
self.__logger.info(f'connection lost: {exc}')
super().connection_lost(exc)
@classmethod
def create_factory(cls, url: str):
def factory(*args, **kwargs):
return cls(*args, url=url, **kwargs)
return factory
async def test_asyncio(ssl_context):
loop = asyncio.get_event_loop()
url = 'https://s3.ap-southeast-1.amazonaws.com/archpi.dabase.com/doesnotexist'
url_parts = urlparse(url)
port = url_parts.port or (80 if url_parts.scheme == 'http' else 443)
infos = await loop.getaddrinfo(url_parts.hostname, port, family=socket.AF_INET)
family, type, proto, canonname, sockaddr = infos[0]
await loop.create_connection(HttpClient.create_factory(url), sockaddr[0], port, ssl=ssl_context, family=family, proto=proto, flags=socket.AI_NUMERICHOST, server_hostname=url_parts.hostname, local_addr=None)
async def asyncio_test():
ssl_context = ssl.create_default_context()
while True:
await test_asyncio(ssl_context)
await asyncio_test() |
|
Date |
User |
Action |
Args |
2018-09-19 22:41:00 | thehesiod | set | recipients:
+ thehesiod, asvetlov, yselivanov |
2018-09-19 22:41:00 | thehesiod | set | messageid: <1537396860.59.0.956365154283.issue34745@psf.upfronthosting.co.za> |
2018-09-19 22:41:00 | thehesiod | link | issue34745 messages |
2018-09-19 22:41:00 | thehesiod | create | |
|