Python Generators — Lazy Evaluation and Memory Efficiency

Why Generators?

Generators produce values one at a time, on demand. Instead of loading an entire dataset into memory, generators yield one value at a time — critical for large files and data pipelines.

Basic Generator Function

def count_up(n):
    i = 1
    while i <= n:
        yield i
        i += 1

for num in count_up(5):
    print(num)  # 1, 2, 3, 4, 5

Reading Large Files

def read_large_file(file_path):
    with open(file_path, 'r') as file:
        for line in file:
            yield line.strip()

for line in read_large_file('huge_log.txt'):
    if 'ERROR' in line:
        print(line)

Generator Pipelines

def read_data(source):
    for item in source:
        yield item

def filter_valid(data):
    for item in data:
        if item.get('valid', False):
            yield item

def transform(data):
    for item in data:
        yield {'id': item['id'], 'name': item['name'].upper()}

pipeline = transform(filter_valid(read_data(raw_data)))
for result in pipeline:
    print(result)

yield from — Sub-generators

def flatten(nested_list):
    for item in nested_list:
        if isinstance(item, list):
            yield from flatten(item)
        else:
            yield item

print(list(flatten([1, [2, 3], [4, [5, 6]], 7])))
# [1, 2, 3, 4, 5, 6, 7]