Oh, I got it. Size is too high growing to reach recurtion limit. I replace repr with slow growing data and now there is good old recursion limit exception.

import json

class F:
    counter = 0
    total = 0
    data = b""

    def __call__(self, o):
        self.counter += 1
        self.data = b"1" + self.data
        self.total += len(self.data)
        print(self.counter, len(self.data), self.total)
        return self.data

json.dumps(object(), default=F())