import botocore.session import os import linecache import botocore.exceptions import gc import time import sys import tracemalloc import logging import re import requests print(f"Using: {sys.argv[1]}") CREDENTIAL = "dummy" URLS = { 'https://s3.us-west-2.amazonaws.com/archpi.dabase.com/style.css': { 'method': 'get', 'headers': {'User-Agent': 'Botocore/1.8.21 Python/3.6.4 Darwin/17.5.0', 'X-Amz-Date': '20180518T025044Z', 'X-Amz-Content-SHA256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': f'AWS4-HMAC-SHA256 Credential={CREDENTIAL}/20180518/us-west-2/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=ae552641b9aa9a7a267fcb4e36960cd5863e55d91c9b45fd39b30fdcd2e81489', 'Accept-Encoding': 'identity'} }, 'https://s3.ap-southeast-1.amazonaws.com/archpi.dabase.com/doesnotexist': { 'method': 'GET' if sys.argv[1] == 'get_object' else 'HEAD', 'headers': {'User-Agent': 'Botocore/1.8.21 Python/3.6.4 Darwin/17.5.0', 'X-Amz-Date': '20180518T025221Z', 'X-Amz-Content-SHA256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': f'AWS4-HMAC-SHA256 Credential={CREDENTIAL}/20180518/ap-southeast-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=7a7675ef6d70cb647ed59e02d532ffa80d437fb03976d8246ea9ef102d118794', 'Accept-Encoding': 'identity'} } } def display_top(snapshot): top_stats = snapshot.statistics('traceback') for stat in top_stats: print("%s memory blocks: %.1f KiB" % (stat.count, stat.size / 1024)) for line in stat.traceback.format(): print(line) def test_req(session: requests.Session, url): try: response = session.request(**URLS[url], url=url) if response.status_code >= 400: raise Exception data = response.read() except: pass def test(s3_client): try: method = getattr(s3_client, sys.argv[1]) method(Bucket='archpi.dabase.com', Key='doesnotexist') except botocore.exceptions.ClientError as e: if e.response['ResponseMetadata']['HTTPStatusCode'] != 404: raise def do_get_obj(s3_client): response = s3_client.get_object(Bucket='archpi.dabase.com', Key='style.css') data = response["Body"].read() response["Body"].close() def main(): logging.basicConfig(level=logging.INFO) boto_session = botocore.session.get_session() boto_config = botocore.config.Config(connect_timeout=15, read_timeout=15, max_pool_connections=1) s3_client = boto_session.create_client('s3', config=boto_config) req_session = requests.Session() test_req(req_session, 'https://s3.us-west-2.amazonaws.com/archpi.dabase.com/style.css') # do_get_obj(s3_client) start = time.time() cycles = 0 while True: # test(s3_client) test_req(req_session, 'https://s3.ap-southeast-1.amazonaws.com/archpi.dabase.com/doesnotexist') # to avoid getting listed in tracemalloc results re._cache.clear() gc.collect() cycles += 1 if cycles == 5: tracemalloc.start(40) if cycles == 25: snapshot = tracemalloc.take_snapshot() display_top(snapshot) if time.time() - start > (60 * 5): break if __name__ == '__main__': main()