Memory Leak in a Long-Running Python Service

# Step 1 — Profile memory with tracemalloc
import tracemalloc

tracemalloc.start()

# ... run the service for a while ...

snapshot = tracemalloc.take_snapshot()
top_stats = snapshot.statistics('lineno')

for stat in top_stats[:10]:
    print(stat)
# Output reveals: webhook_handler.py:45 grew by 1.2 GB — the 'seen_ids' set

# The bug — unbounded set accumulation
seen_ids = set()   # Module-level — never cleared!

def process_webhook(event_id: str, payload: dict):
    if event_id in seen_ids:
        return   # Deduplicate
    seen_ids.add(event_id)   # Grows forever!
    handle_payload(payload)

# Fix 1 — Use Redis with TTL for deduplication (production-grade)
import redis

r = redis.Redis()

def process_webhook(event_id: str, payload: dict):
    key = f"webhook:seen:{event_id}"
    if r.exists(key):
        return
    r.setex(key, time=86400, value=1)  # TTL: 24 hours
    handle_payload(payload)

# Fix 2 — Use a bounded LRU cache (in-memory, simple)
from functools import lru_cache
from collections import OrderedDict

class BoundedSet:
    def __init__(self, maxsize=100_000):
        self._data = OrderedDict()
        self._maxsize = maxsize

    def add(self, key):
        if len(self._data) >= self._maxsize:
            self._data.popitem(last=False)  # Evict oldest
        self._data[key] = True

    def __contains__(self, key):
        return key in self._data

seen_ids = BoundedSet(maxsize=100_000)