Author thehesiod
Recipients asvetlov, thehesiod, yselivanov
Date 2018-09-19.22:41:00
SpamBayes Score -1.0
Marked as misclassified Yes
Message-id <1537396860.59.0.956365154283.issue34745@psf.upfronthosting.co.za>
In-reply-to
Content
I've been trying to track down a leak in aiohttp: https://github.com/aio-libs/aiohttp/issues/3010

it seems like this leak now occurs with raw asyncio SSL sockets.

when the gist script is run like so: python3.7 `which mprof` run --interval=1 ~/dev/test_leak.py -test asyncio_test

it slowly leaks memory.  This is effectively doing the following:

URLS = {
    'https://s3.us-west-2.amazonaws.com/archpi.dabase.com/style.css': {
        'method': 'get',
        'headers': {'User-Agent': 'Botocore/1.8.21 Python/3.6.4 Darwin/17.5.0', 'X-Amz-Date': '20180518T025044Z', 'X-Amz-Content-SHA256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': f'AWS4-HMAC-SHA256 Credential={CREDENTIAL}/20180518/us-west-2/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=ae552641b9aa9a7a267fcb4e36960cd5863e55d91c9b45fd39b30fdcd2e81489', 'Accept-Encoding': 'identity'}
    },

    'https://s3.ap-southeast-1.amazonaws.com/archpi.dabase.com/doesnotexist': {
        'method': 'GET' if sys.argv[1] == 'get_object' else 'HEAD',
        'headers': {'User-Agent': 'Botocore/1.8.21 Python/3.6.4 Darwin/17.5.0', 'X-Amz-Date': '20180518T025221Z', 'X-Amz-Content-SHA256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': f'AWS4-HMAC-SHA256 Credential={CREDENTIAL}/20180518/ap-southeast-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=7a7675ef6d70cb647ed59e02d532ffa80d437fb03976d8246ea9ef102d118794', 'Accept-Encoding': 'identity'}
    }
}


class HttpClient(asyncio.streams.FlowControlMixin):
    transport = None

    def __init__(self, *args, **kwargs):
        self.__url = kwargs.pop('url')
        self.__logger = logging.getLogger()
        super().__init__()

    def connection_made(self, transport):
        self.transport = transport

        url_parts = urlparse(self.__url)
        entry = URLS[self.__url]

        body = f'{entry["method"]} {url_parts.path} HTTP/1.1\r\nAccept: */*\r\nHost: {url_parts.hostname}\r\n'
        for name, value in entry['headers'].items():
            body += f'{name}: {value}\r\n'

        body += '\r\n'
        self.transport.write(body.encode('ascii'))
        self.__logger.info(f'data sent: {body}')

    def data_received(self, data):
        self.__logger.info(f'data received: {data}')

        self.transport.close()
        # asyncio.get_event_loop().call_later(1.0, )

    def eof_received(self):
        self.__logger.info('eof_received')

    def connection_lost(self, exc):
        self.__logger.info(f'connection lost: {exc}')
        super().connection_lost(exc)

    @classmethod
    def create_factory(cls, url: str):
        def factory(*args, **kwargs):
            return cls(*args, url=url, **kwargs)

        return factory


async def test_asyncio(ssl_context):
    loop = asyncio.get_event_loop()

    url = 'https://s3.ap-southeast-1.amazonaws.com/archpi.dabase.com/doesnotexist'
    url_parts = urlparse(url)
    port = url_parts.port or (80 if url_parts.scheme == 'http' else 443)
    infos = await loop.getaddrinfo(url_parts.hostname, port, family=socket.AF_INET)
    family, type, proto, canonname, sockaddr = infos[0]
    await loop.create_connection(HttpClient.create_factory(url), sockaddr[0], port, ssl=ssl_context, family=family, proto=proto, flags=socket.AI_NUMERICHOST, server_hostname=url_parts.hostname, local_addr=None)


async def asyncio_test():
    ssl_context = ssl.create_default_context()

    while True:
        await test_asyncio(ssl_context)


await asyncio_test()
History
Date User Action Args
2018-09-19 22:41:00thehesiodsetrecipients: + thehesiod, asvetlov, yselivanov
2018-09-19 22:41:00thehesiodsetmessageid: <1537396860.59.0.956365154283.issue34745@psf.upfronthosting.co.za>
2018-09-19 22:41:00thehesiodlinkissue34745 messages
2018-09-19 22:41:00thehesiodcreate