Python Other
Decorators
Decorators: simple example
- A decorators is that @something just before the declaration of the function.
- Decorators can modify the behavior of functions or can set some meta information about them.
@some_decorator
def some_function():
pass
Decorators - Flask
- In Flask we use decorators to designate function as "routes".
from flask import Flask
app = Flask(__name__)
@app.route("/")
def main():
return "Hello World!"
@app.route("/login")
def login():
return "Showing the login page ..."
FLASK_APP=flask_app flask run
Decorators - Pytest
- In Pytest we can use decorators to add special marks to test functions
- ... or to mark them as fixtures.
import sys
import pytest
@pytest.mark.skipif(sys.platform != 'linux', reason="Linux tests")
def test_linux():
assert True
@pytest.mark.skip(reason="To show we can skip tests without any condition.")
def test_any():
assert True
@pytest.fixture(autouse = True, scope="module")
def module_demo():
print(f"Fixture")
pytest -v
Decorators caching - no cache
- Each call will execute the function and do the (expensive) computation.
def compute(x, y):
print(f"Called with {x} and {y}")
# some long computation here
return x+y
print(compute(2, 3))
print(compute(3, 4))
print(compute(2, 3))
Called with 2 and 3
5
Called with 3 and 4
7
Called with 2 and 3
5
Decorators caching - with cache
-
cache
-
lru_cache
-
By adding the lru_cache decorator we can tell Python to cache the result and save on computation time.
import functools
@functools.lru_cache()
def compute(x, y):
print(f"Called with {x} and {y}")
# some long computation here
return x+y
print(compute(2, 3))
print(compute(3, 4))
print(compute(2, 3))
Called with 2 and 3
5
Called with 3 and 4
7
5
LRU - Least recently used cache
- LRU - Cache replacement policy
- When we call the function with (1, 5) it removes the least recently used results of (1, 2)
- So next time it has to be computed again.
import functools
@functools.lru_cache(maxsize=3)
def compute(x, y):
print(f"Called with {x} and {y}")
# some long computation here
return x+y
compute(1, 2) # Called with 1 and 2
compute(1, 2)
compute(1, 2)
compute(1, 3) # Called with 1 and 3
compute(1, 3)
compute(1, 4) # Called with 1 and 4
compute(1, 4)
compute(1, 5) # Called with 1 and 5
compute(1, 2) # Called with 1 and 2
compute(1, 2)
LRU - Least recently used cache
- Here we called (1, 2) after (1, 4) when it was still in the cache
- When we called (1, 5) it removed the LRU pair, but it was NOT the (1, 2) pair
- So it was in the cache even after the (1, 5) call.
import functools
@functools.lru_cache(maxsize=3)
def compute(x, y):
print(f"Called with {x} and {y}")
# some long computation here
return x+y
compute(1, 2) # Called with 1 and 2
compute(1, 2)
compute(1, 2)
compute(1, 3) # Called with 1 and 3
compute(1, 3)
compute(1, 4) # Called with 1 and 4
compute(1, 4)
compute(1, 2)
compute(1, 5) # Called with 1 and 5
compute(1, 2)
OOP - classmethod - staticmethod
class Person(object):
def __init__(self, name):
print(f"init: '{self}' '{self.__class__.__name__}'")
self.name = name
def show_name(self):
print(f"instance method: '{self}' '{self.__class__.__name__}'")
@classmethod
def from_occupation(cls, occupation):
print(f"class method '{cls}' '{cls.__class__.__name__}'")
@staticmethod
def is_valid_occupation(param):
print(f"static method '{param}' '{param.__class__.__name__}'")
fb = Person('Foo Bar')
fb.show_name()
fb.from_occupation('Tailor')
Person.from_occupation('Tailor') # This is how we should call it.
fb.is_valid_occupation('Tailor')
Person.is_valid_occupation('Tailor')
init: '<__main__.Person object at 0x7fb008f3a640>' 'Person'
instance method: '<__main__.Person object at 0x7fb008f3a640>' 'Person'
class method '<class '__main__.Person'>' 'type'
class method '<class '__main__.Person'>' 'type'
static method 'Tailor' 'str'
static method 'Tailor' 'str'
Use cases for decorators in Python
-
classmethod
-
staticmethod
-
pytest
-
Common decorators are @classmethod and @staticmethod.
-
Flask uses them to mark and configure the routes.
-
Pytest uses them to add marks to the tests.
-
Logging calls with parameters.
-
Logging elapsed time of calls.
-
Access control in Django or other web frameworks. (e.g. login required)
-
Memoization (caching)
-
Retry
-
Function timeout
-
Locking for thread safety
Function assignment
Before we learn about decorators let's remember that we can assign function names to other names and then use the new name:
def hello(name):
print(f"Hello {name}")
hello("Python")
print(hello)
greet = hello
greet("Python")
print(greet)
Hello Python
<function hello at 0x7f8aee3401f0>
Hello Python
<function hello at 0x7f8aee3401f0>
Function assignment - alias print to say
say = print
say("Hello World")
Function assignment - don't do this
numbers = [2, 4, 3, 1, 1, 1]
print(sum(numbers)) # 12
print(max(numbers)) # 4
sum = max
print(sum(numbers)) # 4
print(max(numbers)) # 4
sum = lambda values: len(values)
print(sum(numbers)) # 6
Passing functions as parameters
def call(func):
return func(42)
def double(val):
print(2*val)
call(double) # 84
call(lambda x: print(x // 2)) # 21
Traversing directory tree
import sys
import os
def walker(path, todo):
if os.path.isdir(path):
items = os.listdir(path)
for item in items:
walker(os.path.join(path, item), todo)
else:
todo(path)
def print_size(name):
print(f"{os.stat(name).st_size:6} {name} ")
if __name__ == '__main__':
if len(sys.argv) < 2:
exit(f"Usage: {sys.argv[0]} PATH")
walker(sys.argv[1], print)
#walker(sys.argv[1], print_size)
#walker(sys.argv[1], lambda name: print(f"{os.stat(name).st_size:6} {name[::-1]} "))
Declaring Functions inside other function
Let's also remember that we can define a function inside another function and then the internally defined function only exists in the scope of the function where it was defined in. Not outside.
def f():
def g():
print("in g")
print("start f")
g()
print("end f")
f()
g()
start f
in g
end f
Traceback (most recent call last):
File "examples/decorators/function_in_function.py", line 9, in <module>
g()
NameError: name 'g' is not defined
Returning a new function from a function
def create_function():
print("creating a function")
def internal():
print("This is the generated function")
print("creation done")
return internal
func = create_function()
func()
creating a function
creation done
This is the generated function
Returning a closure
def create_incrementer(num):
def inc(val):
return num + val
return inc
inc_5 = create_incrementer(5)
print(inc_5(10)) # 15
print(inc_5(0)) # 5
inc_7 = create_incrementer(7)
print(inc_7(10)) # 17
print(inc_7(0)) # 7
Decorator
-
@
-
A function that changes the behaviour of other functions.
-
The input of a decorator is a function.
-
The returned value of a decorator is a modified version of the same function.
from some_module import some_decorator
@some_decorator
def f(...):
...
def f(...):
...
f = some_decorator(f)
Decorator Demo
- Just a simple example created step-by-step
import time
def replace(func):
def new_func():
print("start new")
start = time.time()
func()
end = time.time()
print(f"end new {end-start}")
return new_func
@replace
def f():
time.sleep(1)
print("in f")
f()
Decorator to register function
- Pytest, Flask probably do this
functions = []
def register(func):
global functions
functions.append(func.__name__)
return func
@register
def f():
print("in f")
print(functions)
A recursive Fibonacci
def fibo(n):
if n in (1,2):
return 1
return fibo(n-1) + fibo(n-2)
print(fibo(5)) # 5
trace fibo
import decor
@decor.tron
def fibo(n):
if n in (1,2):
return 1
return fibo(n-1) + fibo(n-2)
print(fibo(5))
Calling fibo(5)
Calling fibo(4)
Calling fibo(3)
Calling fibo(2)
Calling fibo(1)
Calling fibo(2)
Calling fibo(3)
Calling fibo(2)
Calling fibo(1)
5
tron decorator
def tron(func):
def new_func(v):
print(f"Calling {func.__name__}({v})")
return func(v)
return new_func
Decorate with direct call
import decor
def fibo(n):
if n in (1,2):
return 1
return fibo(n-1) + fibo(n-2)
fibo = decor.tron(fibo)
print(fibo(5))
Decorate with parameter
import decor_param
@decor_param.tron('foo')
def fibo(n):
if n in (1,2):
return 1
return fibo(n-1) + fibo(n-2)
print(fibo(5))
foo Calling fibo(5)
foo Calling fibo(4)
foo Calling fibo(3)
foo Calling fibo(2)
foo Calling fibo(1)
foo Calling fibo(2)
foo Calling fibo(3)
foo Calling fibo(2)
foo Calling fibo(1)
5
Decorator accepting parameter
def tron(prefix):
def real_tron(func):
def new_func(v):
print("{} Calling {}({})".format(prefix, func.__name__, v))
return func(v)
return new_func
return real_tron
Decorate function with any signature
- How can we decorate a function that is flexible on the number of arguments?
- Accept
*argsand**kwargsand pass them on.
from decor_any import tron
@tron
def one(param):
print(f"one({param})")
@tron
def two(first, second = 42):
print(f"two({first}, {second})")
one("hello")
one(param = "world")
two("hi")
two(first = "Foo", second = "Bar")
Decorate function with any signature - implementation
def tron(func):
def new_func(*args, **kw):
params = list(map(lambda p: str(p), args))
for (k, v) in kw.items():
params.append(f"{k}={v}")
print("Calling {}({})".format(func.__name__, ', '.join(params)))
return func(*args, **kw)
return new_func
Calling one(hello)
one(hello)
Calling one(param=world)
one(world)
Calling two(hi)
two(hi, 42)
Calling two(first=Foo, second=Bar)
two(Foo, Bar)
Decorate function with any signature - skeleton
def decorator(func):
def wrapper(*args, **kw):
return func(*args, **kw)
return wrapper
@decorator
def zero():
print("zero")
@decorator
def one(x):
print(f"one({x})")
@decorator
def two(x, y):
print(f"two({x, y})")
zero()
one('hello')
two( y = 7, x = 8 )
print(zero)
print(one)
print(two)
print(zero.__name__)
print(one.__name__)
print(two.__name__)
zero
one(hello)
two((8, 7))
<function decorator.<locals>.wrapper at 0x7f1165258a60>
<function decorator.<locals>.wrapper at 0x7f1165258b80>
<function decorator.<locals>.wrapper at 0x7f1165258ca0>
Decorate function with any signature - skeleton with name
import functools
def decorator(func):
@functools.wraps(func)
def wrapper(*args, **kw):
return func(*args, **kw)
return wrapper
@decorator
def zero():
print("zero")
@decorator
def one(x):
print(f"one({x})")
@decorator
def two(x, y):
print(f"two({x, y})")
zero()
one('hello')
two( y = 7, x = 8 )
print(zero)
print(one)
print(two)
print(zero.__name__)
print(one.__name__)
print(two.__name__)
zero
one(hello)
two((8, 7))
<function zero at 0x7f9079bdca60>
<function one at 0x7f9079bdcb80>
<function two at 0x7f9079bdcca0>
Functool - partial
- partial
from functools import partial
val = '101010'
print(int(val, base=2))
basetwo = partial(int, base=2)
basetwo.__doc__ = 'Convert base 2 string to an int.'
print(basetwo(val))
# Based on example from https://docs.python.org/3/library/functools.html
Exercise: Logger decorator
- In the previous pages we created a decorator that can decorate arbitrary function logging the call and its parameters.
- Add time measurement to each call to see how long each function took.
Exercise: decorators decorator
Write a function that gets a functions as attribute and returns a new functions while memoizing (caching) the input/output pairs. Then write a unit test that checks it. You probably will need to create a subroutine to be decoratorsd.
- Write tests for the fibonacci functions.
- Implement the decorators decorator for a function with a single parameter.
- Apply the decorator.
- Run the tests again.
- Check the speed differences.
- or decorate with tron to see the calls...
Solution: Logger decorator
import time
def tron(func):
def new_func(*args, **kwargs):
start = time.time()
print("Calling {}({}, {})".format(func.__name__, args, kwargs))
out = func(*args, **kwargs)
end = time.time()
print("Finished {}({})".format(func.__name__, out))
print("Elapsed time: {}".format(end - start))
return out
return new_func
Solution: Logger decorator (testing)
from logger_decor import tron
@tron
def f(a, b=1, *args, **kwargs):
print('a: ', a)
print('b: ', b)
print('args: ', args)
print('kwargs:', kwargs)
return a + b
f(2, 3, 4, 5, c=6, d=7)
print()
f(2, c=5, d=6)
print()
f(10)
Calling f((2, 3, 4, 5), {'c': 6, 'd': 7})
a: 2
b: 3
args: (4, 5)
kwargs: {'c': 6, 'd': 7}
Finished f(5)
Elapsed time: 1.3589859008789062e-05
Calling f((2,), {'c': 5, 'd': 6})
a: 2
b: 1
args: ()
kwargs: {'c': 5, 'd': 6}
Finished f(3)
Elapsed time: 5.245208740234375e-06
Calling f((10,), {})
a: 10
b: 1
args: ()
kwargs: {}
Finished f(11)
Elapsed time: 4.291534423828125e-06
Solution decorators decorator
import sys
import memoize_attribute
import memoize_nonlocal
import decor_any
#@memoize_attribute.memoize
#@memoize_nonlocal.memoize
#@decor_any.tron
def fibonacci(n):
if n == 1:
return 1
if n == 2:
return 1
return fibonacci(n-1) + fibonacci(n-2)
if __name__ == '__main__':
if len(sys.argv) != 2:
sys.stderr.write("Usage: {} N\n".format(sys.argv[0]))
exit(1)
print(fibonacci(int(sys.argv[1])))
def memoize(f):
data = {}
def caching(n):
nonlocal data
key = n
if key not in data:
data[key] = f(n)
return data[key]
return caching
def memoize(f):
def caching(n):
key = n
#if 'data' not in caching.__dict__:
# caching.data = {}
if key not in caching.data:
caching.data[key] = f(n)
return caching.data[key]
caching.data = {}
return caching
Before
$ time python fibonacci.py 35
9227465
real 0m3.850s
user 0m3.832s
sys 0m0.015s
After
$ time python fibonacci.py 35
9227465
real 0m0.034s
user 0m0.019s
sys 0m0.014s
A list of functions
def hello(name):
print(f"Hello {name}")
def morning(name):
print(f"Good morning {name}")
hello("Jane")
morning("Jane")
print()
funcs = [hello, morning]
funcs[0]("Peter")
print()
for func in funcs:
func("Mary")
Hello Jane
Good morning Jane
Hello Peter
Hello Mary
Good morning Mary
Insert element in sorted list using insort
- insort
import bisect
solar_system = ['Earth', 'Jupiter', 'Mercury', 'Saturn', 'Venus']
name = 'Mars'
# Find the location where to insert the element to keep the list sorted and insert the element
bisect.insort(solar_system, name)
print(solar_system)
print(sorted(solar_system))
import sys
import os
def traverse(path):
if os.path.isfile(path):
print(path)
return
if os.path.isdir(path):
for item in os.listdir(path):
traverse(os.path.join(path, item))
return
# other unhandled things
if len(sys.argv) < 2:
exit(f"Usage: {sys.argv[0]} DIR|FILE")
traverse(sys.argv[1])
import sys
import os
def traverse(path, func):
response = {}
if os.path.isfile(path):
func(path)
return response
if os.path.isdir(path):
for item in os.listdir(path):
traverse(os.path.join(path, item), func)
return response
# other unhandled things
if len(sys.argv) < 2:
exit(f"Usage: {sys.argv[0]} DIR|FILE")
#traverse(sys.argv[1], print)
#traverse(sys.argv[1], lambda path: print(f"{os.path.getsize(path):>6} {path}"))
import sys
import os
def traverse(path, func):
if os.path.isfile(path):
func(path)
return
if os.path.isdir(path):
for item in os.listdir(path):
traverse(os.path.join(path, item), func)
return
# other unhandled things
if len(sys.argv) < 2:
exit(f"Usage: {sys.argv[0]} DIR|FILE")
#traverse(sys.argv[1], print)
#traverse(sys.argv[1], lambda path: print(f"{os.path.getsize(path):>6} {path}"))
#from inspect import getmembers, isfunction
import inspect
def change(sub):
def new(*args, **kw):
print("before")
res = sub(*args, **kw)
print("after")
return res
return new
def add(x, y):
return x+y
#print(add(2, 3))
fixed = change(add)
#print(fixed(3, 4))
def replace(subname):
def new(*args, **kw):
print("before")
res = locals()[subname](*args, **kw)
print("after")
return res
locals()[subname] = new
replace('add')
add(1, 7)
def say():
print("hello")
#print(dir())
#getattr('say')