From f0675dcb6dcc717e37c7c85133362085fd90428d Mon Sep 17 00:00:00 2001 From: Miki Tebeka Date: Mon, 26 Aug 2024 09:50:28 +0300 Subject: [PATCH] caching in python --- py-cache/cache.py | 15 +++++++++++++++ py-cache/caching.md | 7 +++++++ py-cache/current.txt | 1 + py-cache/end.ipy | 6 ++++++ py-cache/four.md | 3 +++ py-cache/header.md | 3 +++ py-cache/libs.md | 14 ++++++++++++++ py-cache/next.ipy | 39 +++++++++++++++++++++++++++++++++++++++ py-cache/playbook.md | 5 +++++ py-cache/setup.ipy | 5 +++++ py-cache/three.md | 3 +++ py-cache/two.md | 3 +++ py-cache/user.py | 14 ++++++++++++++ py-cache/user_cached.py | 17 +++++++++++++++++ py-cache/user_lru.py | 17 +++++++++++++++++ py-cache/user_mem.py | 19 +++++++++++++++++++ 16 files changed, 171 insertions(+) create mode 100644 py-cache/cache.py create mode 100644 py-cache/caching.md create mode 100644 py-cache/current.txt create mode 100644 py-cache/end.ipy create mode 100644 py-cache/four.md create mode 100644 py-cache/header.md create mode 100644 py-cache/libs.md create mode 100644 py-cache/next.ipy create mode 100644 py-cache/playbook.md create mode 100644 py-cache/setup.ipy create mode 100644 py-cache/three.md create mode 100644 py-cache/two.md create mode 100644 py-cache/user.py create mode 100644 py-cache/user_cached.py create mode 100644 py-cache/user_lru.py create mode 100644 py-cache/user_mem.py diff --git a/py-cache/cache.py b/py-cache/cache.py new file mode 100644 index 0000000..e9456c3 --- /dev/null +++ b/py-cache/cache.py @@ -0,0 +1,15 @@ +from functools import wraps + + +def cached(fn): + cache = {} # args -> value + + @wraps(fn) + def wrapper(*args, **kw): + if args in cache: + return cache[args] + + value = cache[args] = fn(*args) + return value + + return wrapper diff --git a/py-cache/caching.md b/py-cache/caching.md new file mode 100644 index 0000000..4d84359 --- /dev/null +++ b/py-cache/caching.md @@ -0,0 +1,7 @@ +# Caching + +Save a computation result and using it later. AKA memoization. + +You trade memory consumption for performance. + +Works on "pure" functions that don't have side effects. diff --git a/py-cache/current.txt b/py-cache/current.txt new file mode 100644 index 0000000..ca7bf83 --- /dev/null +++ b/py-cache/current.txt @@ -0,0 +1 @@ +13 \ No newline at end of file diff --git a/py-cache/end.ipy b/py-cache/end.ipy new file mode 100644 index 0000000..84c7963 --- /dev/null +++ b/py-cache/end.ipy @@ -0,0 +1,6 @@ +import os + +!kitty +icat book-covers.png +width = os.get_terminal_size().columns +url = 'https://pragprog.com/search/?q=miki+tebeka' +print(f'{url:^{width}}') diff --git a/py-cache/four.md b/py-cache/four.md new file mode 100644 index 0000000..b1c507e --- /dev/null +++ b/py-cache/four.md @@ -0,0 +1,3 @@ +There's two hard problems in computer science: We only have one joke and it's not funny. + +- Phillip Scott Bowden diff --git a/py-cache/header.md b/py-cache/header.md new file mode 100644 index 0000000..437aba8 --- /dev/null +++ b/py-cache/header.md @@ -0,0 +1,3 @@ +# Caching in Python +[Miki Tebeka](mailto:miki@353solutions.com) + diff --git a/py-cache/libs.md b/py-cache/libs.md new file mode 100644 index 0000000..bc8c9ac --- /dev/null +++ b/py-cache/libs.md @@ -0,0 +1,14 @@ +# Use Libraries + + +The "Not Invented Here" (NIH) Syndrome is the tendency to reject ideas that come from "outside". +And unfortunately, "outside" can still apply to solutions that come from other teams within the organization. + +- functools.lru_cache + - https://docs.python.org/3/library/functools.html#functools.lru_cache +- joblib.Memory + - https://joblib.readthedocs.io/en/latest/memory.html#memory +- Redis/Memcached + - https://redis.io/ + - https://memcached.org/ + diff --git a/py-cache/next.ipy b/py-cache/next.ipy new file mode 100644 index 0000000..8565d31 --- /dev/null +++ b/py-cache/next.ipy @@ -0,0 +1,39 @@ +current = 'current.txt' + +try: + with open(current) as fp: + i = int(fp.read()) +except OSError: + i = 0 + +!clear +if i == 0: + !glow caching.md +elif i == 1: + !source-highlight -n -f esc -s python -i user.py + %run user.py +elif i == 2: + !source-highlight -n -f esc -s python -i cache.py +elif i == 3: + !source-highlight -n -f esc -s python -i user_cached.py + %run user_cached.py +elif i == 4: + !glow two.md +elif i == 5: + !glow three.md +elif i == 6: + !glow four.md +elif i == 7: + !glow libs.md +elif i == 8: + !source-highlight -n -f esc -s python -i user_lru.py + %run user_lru.py +elif i == 9: + !source-highlight -n -f esc -s python -i user_mem.py + %run user_mem.py +else: + echo 'Thank You!' | cowsay | lolcat -a -d 3 + + +with open(current, 'w') as out: + print(i+1, file=out, end='') diff --git a/py-cache/playbook.md b/py-cache/playbook.md new file mode 100644 index 0000000..8c958ed --- /dev/null +++ b/py-cache/playbook.md @@ -0,0 +1,5 @@ +%run setup.py + +%timeit get_user('bruce') + +In cache show get_user(['bruce']) diff --git a/py-cache/setup.ipy b/py-cache/setup.ipy new file mode 100644 index 0000000..8384746 --- /dev/null +++ b/py-cache/setup.ipy @@ -0,0 +1,5 @@ +%alias_magic -p next.ipy next %run +%alias_magic -p end.ipy end %run +!rm -f current.txt +!clear +!glow header.md diff --git a/py-cache/three.md b/py-cache/three.md new file mode 100644 index 0000000..36483de --- /dev/null +++ b/py-cache/three.md @@ -0,0 +1,3 @@ +There are 2 hard problems in computer science: cache invalidation, naming things, and off-by-1 errors. + +- Leon Bambrick diff --git a/py-cache/two.md b/py-cache/two.md new file mode 100644 index 0000000..b9c36f9 --- /dev/null +++ b/py-cache/two.md @@ -0,0 +1,3 @@ +There are only two hard things in Computer Science: cache invalidation and naming things. + +- Phil Karlton diff --git a/py-cache/user.py b/py-cache/user.py new file mode 100644 index 0000000..da34b35 --- /dev/null +++ b/py-cache/user.py @@ -0,0 +1,14 @@ +from dataclasses import dataclass +from time import sleep + +@dataclass +class User: + login: str + name: str + roles: list[str] + + +def get_user(login): + sleep(0.1) # simulate DB access + return User(login, 'Bruce', ['ADMIN']) + diff --git a/py-cache/user_cached.py b/py-cache/user_cached.py new file mode 100644 index 0000000..5f73133 --- /dev/null +++ b/py-cache/user_cached.py @@ -0,0 +1,17 @@ +from dataclasses import dataclass +from time import sleep + +from cache import cached + +@dataclass +class User: + login: str + name: str + roles: list[str] + + +@cached +def get_user(login): + sleep(0.1) # simulate DB access + return User(login, 'Bruce', ['ADMIN']) + diff --git a/py-cache/user_lru.py b/py-cache/user_lru.py new file mode 100644 index 0000000..182d57e --- /dev/null +++ b/py-cache/user_lru.py @@ -0,0 +1,17 @@ +from dataclasses import dataclass +from time import sleep + +from functools import lru_cache + +@dataclass +class User: + login: str + name: str + roles: list[str] + + +@lru_cache(1024) +def get_user(login): + sleep(0.1) # simulate DB access + return User(login, 'Bruce', ['ADMIN']) + diff --git a/py-cache/user_mem.py b/py-cache/user_mem.py new file mode 100644 index 0000000..c763de7 --- /dev/null +++ b/py-cache/user_mem.py @@ -0,0 +1,19 @@ +from dataclasses import dataclass +from time import sleep + +from joblib import Memory + +memory = Memory('/tmp/users', verbose=False) + +@dataclass +class User: + login: str + name: str + roles: list[str] + + +@memory.cache +def get_user(login): + sleep(0.1) # simulate DB access + return User(login, 'Bruce', ['ADMIN']) +