1# python-performance-optimization — detailed patterns and worked examples
2 
3## Profiling Tools
4 
5### Pattern 1: cProfile - CPU Profiling
6 
7```python
8import cProfile
9import pstats
10from pstats import SortKey
11 
12def slow_function():
13    """Function to profile."""
14    total = 0
15    for i in range(1000000):
16        total += i
17    return total
18 
19def another_function():
20    """Another function."""
21    return [i**2 for i in range(100000)]
22 
23def main():
24    """Main function to profile."""
25    result1 = slow_function()
26    result2 = another_function()
27    return result1, result2
28 
29# Profile the code
30if __name__ == "__main__":
31    profiler = cProfile.Profile()
32    profiler.enable()
33 
34    main()
35 
36    profiler.disable()
37 
38    # Print stats
39    stats = pstats.Stats(profiler)
40    stats.sort_stats(SortKey.CUMULATIVE)
41    stats.print_stats(10)  # Top 10 functions
42 
43    # Save to file for later analysis
44    stats.dump_stats("profile_output.prof")
45```
46 
47**Command-line profiling:**
48 
49```bash
50# Profile a script
51python -m cProfile -o output.prof script.py
52 
53# View results
54python -m pstats output.prof
55# In pstats:
56# sort cumtime
57# stats 10
58```
59 
60### Pattern 2: line_profiler - Line-by-Line Profiling
61 
62```python
63# Install: pip install line-profiler
64 
65# Add @profile decorator (line_profiler provides this)
66@profile
67def process_data(data):
68    """Process data with line profiling."""
69    result = []
70    for item in data:
71        processed = item * 2
72        result.append(processed)
73    return result
74 
75# Run with:
76# kernprof -l -v script.py
77```
78 
79**Manual line profiling:**
80 
81```python
82from line_profiler import LineProfiler
83 
84def process_data(data):
85    """Function to profile."""
86    result = []
87    for item in data:
88        processed = item * 2
89        result.append(processed)
90    return result
91 
92if __name__ == "__main__":
93    lp = LineProfiler()
94    lp.add_function(process_data)
95 
96    data = list(range(100000))
97 
98    lp_wrapper = lp(process_data)
99    lp_wrapper(data)
100 
101    lp.print_stats()
102```
103 
104### Pattern 3: memory_profiler - Memory Usage
105 
106```python
107# Install: pip install memory-profiler
108 
109from memory_profiler import profile
110 
111@profile
112def memory_intensive():
113    """Function that uses lots of memory."""
114    # Create large list
115    big_list = [i for i in range(1000000)]
116 
117    # Create large dict
118    big_dict = {i: i**2 for i in range(100000)}
119 
120    # Process data
121    result = sum(big_list)
122 
123    return result
124 
125if __name__ == "__main__":
126    memory_intensive()
127 
128# Run with:
129# python -m memory_profiler script.py
130```
131 
132### Pattern 4: py-spy - Production Profiling
133 
134```bash
135# Install: pip install py-spy
136 
137# Profile a running Python process
138py-spy top --pid 12345
139 
140# Generate flamegraph
141py-spy record -o profile.svg --pid 12345
142 
143# Profile a script
144py-spy record -o profile.svg -- python script.py
145 
146# Dump current call stack
147py-spy dump --pid 12345
148```
149 
150## Optimization Patterns
151 
152### Pattern 5: List Comprehensions vs Loops
153 
154```python
155import timeit
156 
157# Slow: Traditional loop
158def slow_squares(n):
159    """Create list of squares using loop."""
160    result = []
161    for i in range(n):
162        result.append(i**2)
163    return result
164 
165# Fast: List comprehension
166def fast_squares(n):
167    """Create list of squares using comprehension."""
168    return [i**2 for i in range(n)]
169 
170# Benchmark
171n = 100000
172 
173slow_time = timeit.timeit(lambda: slow_squares(n), number=100)
174fast_time = timeit.timeit(lambda: fast_squares(n), number=100)
175 
176print(f"Loop: {slow_time:.4f}s")
177print(f"Comprehension: {fast_time:.4f}s")
178print(f"Speedup: {slow_time/fast_time:.2f}x")
179 
180# Even faster for simple operations: map
181def faster_squares(n):
182    """Use map for even better performance."""
183    return list(map(lambda x: x**2, range(n)))
184```
185 
186### Pattern 6: Generator Expressions for Memory
187 
188```python
189import sys
190 
191def list_approach():
192    """Memory-intensive list."""
193    data = [i**2 for i in range(1000000)]
194    return sum(data)
195 
196def generator_approach():
197    """Memory-efficient generator."""
198    data = (i**2 for i in range(1000000))
199    return sum(data)
200 
201# Memory comparison
202list_data = [i for i in range(1000000)]
203gen_data = (i for i in range(1000000))
204 
205print(f"List size: {sys.getsizeof(list_data)} bytes")
206print(f"Generator size: {sys.getsizeof(gen_data)} bytes")
207 
208# Generators use constant memory regardless of size
209```
210 
211### Pattern 7: String Concatenation
212 
213```python
214import timeit
215 
216def slow_concat(items):
217    """Slow string concatenation."""
218    result = ""
219    for item in items:
220        result += str(item)
221    return result
222 
223def fast_concat(items):
224    """Fast string concatenation with join."""
225    return "".join(str(item) for item in items)
226 
227def faster_concat(items):
228    """Even faster with list."""
229    parts = [str(item) for item in items]
230    return "".join(parts)
231 
232items = list(range(10000))
233 
234# Benchmark
235slow = timeit.timeit(lambda: slow_concat(items), number=100)
236fast = timeit.timeit(lambda: fast_concat(items), number=100)
237faster = timeit.timeit(lambda: faster_concat(items), number=100)
238 
239print(f"Concatenation (+): {slow:.4f}s")
240print(f"Join (generator): {fast:.4f}s")
241print(f"Join (list): {faster:.4f}s")
242```
243 
244### Pattern 8: Dictionary Lookups vs List Searches
245 
246```python
247import timeit
248 
249# Create test data
250size = 10000
251items = list(range(size))
252lookup_dict = {i: i for i in range(size)}
253 
254def list_search(items, target):
255    """O(n) search in list."""
256    return target in items
257 
258def dict_search(lookup_dict, target):
259    """O(1) search in dict."""
260    return target in lookup_dict
261 
262target = size - 1  # Worst case for list
263 
264# Benchmark
265list_time = timeit.timeit(
266    lambda: list_search(items, target),
267    number=1000
268)
269dict_time = timeit.timeit(
270    lambda: dict_search(lookup_dict, target),
271    number=1000
272)
273 
274print(f"List search: {list_time:.6f}s")
275print(f"Dict search: {dict_time:.6f}s")
276print(f"Speedup: {list_time/dict_time:.0f}x")
277```
278 
279### Pattern 9: Local Variable Access
280 
281```python
282import timeit
283 
284# Global variable (slow)
285GLOBAL_VALUE = 100
286 
287def use_global():
288    """Access global variable."""
289    total = 0
290    for i in range(10000):
291        total += GLOBAL_VALUE
292    return total
293 
294def use_local():
295    """Use local variable."""
296    local_value = 100
297    total = 0
298    for i in range(10000):
299        total += local_value
300    return total
301 
302# Local is faster
303global_time = timeit.timeit(use_global, number=1000)
304local_time = timeit.timeit(use_local, number=1000)
305 
306print(f"Global access: {global_time:.4f}s")
307print(f"Local access: {local_time:.4f}s")
308print(f"Speedup: {global_time/local_time:.2f}x")
309```
310 
311### Pattern 10: Function Call Overhead
312 
313```python
314import timeit
315 
316def calculate_inline():
317    """Inline calculation."""
318    total = 0
319    for i in range(10000):
320        total += i * 2 + 1
321    return total
322 
323def helper_function(x):
324    """Helper function."""
325    return x * 2 + 1
326 
327def calculate_with_function():
328    """Calculation with function calls."""
329    total = 0
330    for i in range(10000):
331        total += helper_function(i)
332    return total
333 
334# Inline is faster due to no call overhead
335inline_time = timeit.timeit(calculate_inline, number=1000)
336function_time = timeit.timeit(calculate_with_function, number=1000)
337 
338print(f"Inline: {inline_time:.4f}s")
339print(f"Function calls: {function_time:.4f}s")
340```
341 
342For advanced optimization techniques including NumPy vectorization, caching, memory management, parallelization, async I/O, database optimization, and benchmarking tools, see [references/advanced-patterns.md](references/advanced-patterns.md)
343