🔄 Using Generators
Generators are memory-efficient functions that produce values one at a time instead of creating entire lists in memory. They're perfect for processing large datasets, streaming data, or creating infinite sequences.
# Simple generator function
def count_up_to(max_num):
count = 1
while count <= max_num:
yield count
count += 1
# Generator expression
squares = (x**2 for x in range(5))
# Using generators
counter = count_up_to(3)
for number in counter:
print(f"Count: {number}")
# Convert generator to list (if needed)
square_list = list(squares)
print(f"Squares: {square_list}")
🎯 Understanding Generators
Generators use the yield
keyword to produce values on-demand, saving memory by not storing all values at once.
Simple Generator Functions
def simple_generator():
yield 1
yield 2
yield 3
def fibonacci(limit):
a, b = 0, 1
while a < limit:
yield a
a, b = b, a + b
def read_lines():
lines = ["First line", "Second line", "Third line"]
for line in lines:
yield line.strip()
# Using generators
gen = simple_generator()
print(list(gen)) # [1, 2, 3]
fib_numbers = list(fibonacci(20))
print(f"Fibonacci: {fib_numbers}")
for line in read_lines():
print(f"Line: {line}")
Generator Expressions
# Generator expressions (like list comprehensions but with parentheses)
numbers = (x for x in range(10))
squares = (x**2 for x in range(5))
even_squares = (x**2 for x in range(10) if x % 2 == 0)
# Using generator expressions
print("Numbers:", list(numbers))
print("Squares:", list(squares))
print("Even squares:", list(even_squares))
# Generator for processing data
data = [1, 2, 3, 4, 5]
doubled = (x * 2 for x in data)
filtered = (x for x in doubled if x > 4)
print("Filtered doubled:", list(filtered))
📋 Generator vs List Comparison
Feature | Generator | List |
---|---|---|
Memory usage | Low (one item at a time) | High (all items stored) |
Speed | Fast startup | Slower for large data |
Reusability | One-time use | Reusable |
Indexing | No random access | Full indexing support |
Best for | Large/infinite data | Small datasets, random access |
🔧 Practical Generator Examples
File Processing Generator
def process_file_lines(filename=None):
# Simulate file reading
sample_lines = [
"apple,5",
"banana,3",
"orange,8",
"grape,12"
]
for line in sample_lines:
if ',' in line:
name, count = line.strip().split(',')
yield {
'name': name,
'count': int(count)
}
def filter_high_count(items, threshold=5):
for item in items:
if item['count'] > threshold:
yield item
# Chain generators together
file_data = process_file_lines()
high_count_items = filter_high_count(file_data, threshold=5)
print("High count items:")
for item in high_count_items:
print(f" {item['name']}: {item['count']}")
Number Sequence Generators
def even_numbers(start=0, end=None):
current = start if start % 2 == 0 else start + 1
while end is None or current <= end:
yield current
current += 2
if end is not None and current > end:
break
def prime_numbers(limit):
def is_prime(n):
if n < 2:
return False
for i in range(2, int(n**0.5) + 1):
if n % i == 0:
return False
return True
for num in range(2, limit + 1):
if is_prime(num):
yield num
# Use the generators
evens = list(even_numbers(1, 20))
primes = list(prime_numbers(30))
print(f"Even numbers 1-20: {evens}")
print(f"Prime numbers up to 30: {primes}")
Data Processing Pipeline
def load_data():
# Simulate loading data
raw_data = [
" Alice,30,Engineer ",
" Bob,25,Designer ",
" Charlie,35,Manager ",
" Diana,28,Developer "
]
for item in raw_data:
yield item
def clean_data(data_gen):
for item in data_gen:
cleaned = item.strip()
if cleaned:
yield cleaned
def parse_data(data_gen):
for item in data_gen:
parts = item.split(',')
if len(parts) == 3:
yield {
'name': parts[0],
'age': int(parts[1]),
'role': parts[2]
}
def filter_by_age(data_gen, min_age=25):
for person in data_gen:
if person['age'] >= min_age:
yield person
# Create data processing pipeline
raw = load_data()
cleaned = clean_data(raw)
parsed = parse_data(cleaned)
filtered = filter_by_age(parsed, min_age=28)
print("Filtered results:")
for person in filtered:
print(f" {person['name']} ({person['age']}) - {person['role']}")
📊 Generator Methods Reference
Method | Purpose | Example |
---|---|---|
next() | Get next value | next(generator) |
list() | Convert to list | list(generator) |
sum() | Sum all values | sum(number_gen) |
max() | Find maximum | max(number_gen) |
🔄 Generator State and Memory
def stateful_generator():
print("Starting generator")
yield 1
print("After first yield")
yield 2
print("After second yield")
yield 3
print("Generator finished")
def memory_efficient_range(n):
# This uses constant memory regardless of n
current = 0
while current < n:
yield current
current += 1
# Test stateful generator
print("Testing stateful generator:")
gen = stateful_generator()
print(f"First: {next(gen)}")
print(f"Second: {next(gen)}")
print(f"Third: {next(gen)}")
# Memory efficiency demonstration
print("\nMemory efficient range:")
large_range = memory_efficient_range(1000000)
first_five = []
for i, value in enumerate(large_range):
if i >= 5:
break
first_five.append(value)
print(f"First 5 from large range: {first_five}")
Infinite Generators
def infinite_counter(start=0):
current = start
while True:
yield current
current += 1
def cycle_through_list(items):
while True:
for item in items:
yield item
# Use infinite generators (carefully!)
counter = infinite_counter(10)
first_ten = [next(counter) for _ in range(10)]
print(f"Counter from 10: {first_ten}")
colors = cycle_through_list(['red', 'green', 'blue'])
color_sequence = [next(colors) for _ in range(8)]
print(f"Color cycle: {color_sequence}")
Generator Comprehensions in Practice
# Data transformation with generators
data = [
{'name': 'Alice', 'score': 85},
{'name': 'Bob', 'score': 92},
{'name': 'Charlie', 'score': 78},
{'name': 'Diana', 'score': 96}
]
# Generator expressions for data processing
high_scores = (item for item in data if item['score'] > 80)
score_messages = (f"{item['name']}: {item['score']}" for item in high_scores)
print("High score messages:")
for message in score_messages:
print(f" {message}")
# Chaining multiple generators
numbers = range(20)
evens = (x for x in numbers if x % 2 == 0)
squares = (x**2 for x in evens)
large_squares = (x for x in squares if x > 50)
print(f"Large even squares: {list(large_squares)}")
🎯 Key Takeaways
🚀 What's Next?
Learn how to organize your code with modules and packages for better project structure.
Continue to: Work with Modules and Packages
Was this helpful?
Track Your Learning Progress
Sign in to bookmark tutorials and keep track of your learning journey.
Your progress is saved automatically as you read.