From ad4f99d92c2a5b3f1cfa7422bda73b5555f71293 Mon Sep 17 00:00:00 2001 From: Stefan Werner Date: Sun, 29 Oct 2023 13:14:25 +0100 Subject: [PATCH] cc --- Makefile | 23 ++++ classifier/Dockerfile | 2 +- compose.yml | 11 +- integrator/Dockerfile | 10 +- integrator/main.py | 2 +- integrator/requirements.txt | 8 +- integrator/server.py | 16 ++- integrator/setup.py | 9 ++ integrator/states.py | 21 ++++ ...7b38af5421469a767678007a8a525d911-meta.pkl | Bin 0 -> 72 bytes ...7b38af5421469a767678007a8a525d911-text.pkl | Bin 0 -> 1555 bytes ...1fb22f961118205f52433ec500d8ec527-meta.pkl | Bin 0 -> 81 bytes ...1fb22f961118205f52433ec500d8ec527-text.pkl | Bin 0 -> 3640 bytes integrator/texts/shell_builtins.txt | 61 +++++++++++ integrator/texts/strange_concepts.txt | 100 ++++++++++++++++++ integrator/tree.py | 61 +++++++---- linker/Dockerfile | 2 +- service/Dockerfile | 3 +- service/requirements.txt | 2 +- web/src/App.css | 1 - web/src/ui/MobileControls.jsx | 7 +- web/src/ui/Puzzle.jsx | 50 +++++++-- web/src/ui/editor/Controls.jsx | 77 ++++++++++++++ web/src/ui/editor/Editor.jsx | 5 +- web/src/ui/editor/PuzzleView.jsx | 39 ++++++- worker/Dockerfile | 2 +- worker/tasks.py | 2 +- 27 files changed, 458 insertions(+), 56 deletions(-) create mode 100644 integrator/setup.py create mode 100644 integrator/states/1cb10b667a508d3585b760a6ef9e8567b38af5421469a767678007a8a525d911-meta.pkl create mode 100644 integrator/states/1cb10b667a508d3585b760a6ef9e8567b38af5421469a767678007a8a525d911-text.pkl create mode 100644 integrator/states/dcea87b9610d7c1e4d43d5003ceef8c1fb22f961118205f52433ec500d8ec527-meta.pkl create mode 100644 integrator/states/dcea87b9610d7c1e4d43d5003ceef8c1fb22f961118205f52433ec500d8ec527-text.pkl create mode 100644 integrator/texts/shell_builtins.txt create mode 100644 integrator/texts/strange_concepts.txt create mode 100644 web/src/ui/editor/Controls.jsx diff --git a/Makefile b/Makefile index 2822260..30ca311 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,8 @@ -include .env +ifneq ($(ENV_FILE),) + include $(ENV_FILE) + export +endif export COMPOSE_PROFILES=default @@ -14,6 +18,25 @@ start: docker compose up -d +build: + eval $(minikube docker-env) &&docker compose down -v + BUILDKIT_PROGRESS=plain docker compose build + +kompose: + $(MAKE) ENV_FILE=.env _kompose +_kompose: + kompose convert -f compose.yml -o k8s/ + +apply: + eval $$(minikube docker-env) && kubectl apply -f k8s/ + +minikube-add-images: + eval $$(minikube docker-env) && docker build -t worker:latest worker/. + eval $$(minikube docker-env) && docker build -t queue:latest worker/. + eval $$(minikube docker-env) && docker build -t web:latest web/. + eval $$(minikube docker-env) && docker build -t classifier:latest classifier/. + eval $$(minikube docker-env) && docker build -t integrator:latest integrator/. + stop: docker compose down -v diff --git a/classifier/Dockerfile b/classifier/Dockerfile index 30af763..c10b5b5 100644 --- a/classifier/Dockerfile +++ b/classifier/Dockerfile @@ -1,5 +1,5 @@ # Use an official Python runtime as a parent image -FROM kalilinux/kali-rolling:latest +FROM python:3.11-slim-buster ENV PYTHONUNBUFFERED 1 USER root diff --git a/compose.yml b/compose.yml index 66a0d62..a5f9234 100644 --- a/compose.yml +++ b/compose.yml @@ -1,6 +1,7 @@ version: "3" services: nginx: + image: nginx:latest container_name: nginx restart: unless-stopped @@ -27,6 +28,7 @@ services: - ./.certs:$CERTS server: + image: server logging: driver: "json-file" @@ -57,6 +59,7 @@ services: - "./lib:/app/lib" service: + image: service logging: driver: "json-file" @@ -91,6 +94,7 @@ services: - "./.caches/.cache-cr:/chroma:rw" - "./.caches/.cache-hf:/.cache:rw" integrator: + image: integrator:latest logging: driver: "json-file" @@ -107,7 +111,6 @@ services: SYSTEM: /system restart: unless-stopped - image: integrator container_name: integrator env_file: @@ -137,6 +140,7 @@ services: - "./.caches/.cache-mpl:/.config:rw" linker: + image: linker:latest profiles: - linker mem_limit: 10g @@ -174,6 +178,8 @@ services: classifier: + image: classifier + profiles: - classifier mem_limit: 10g @@ -210,7 +216,7 @@ services: web: - + image: web:latest logging: driver: "json-file" options: @@ -261,6 +267,7 @@ services: - "./.caches/.cache-mpl:/.config:rw" queue: + image: queue:latest build: worker command: ["python", "app.py"] ports: diff --git a/integrator/Dockerfile b/integrator/Dockerfile index d81cf4b..2549364 100644 --- a/integrator/Dockerfile +++ b/integrator/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim-buster +FROM pypy:latest ENV PYTHONUNBUFFERED 1 USER root @@ -12,8 +12,10 @@ RUN apt install -y gcc python3-pip RUN pip3 install --upgrade pip RUN pip3 config list RUN pip3 install uwsgi -RUN --mount=type=cache,target=/root/.cache/pip pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu +#RUN --mount=type=cache,target=/root/.cache/pip pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu COPY requirements.txt /app/ +RUN apt install libatlas-base-dev -y +RUN apt install gfortran -y RUN --mount=type=cache,target=/root/.cache/pip pip3 install --no-cache-dir -r requirements.txt RUN mkdir /.cache RUN chmod 777 /.cache @@ -24,5 +26,5 @@ ENV FLASK_ENV=development ENV MPLCONFIGDIR=/config/matplotlib RUN mkdir /matplotlib RUN chmod 777 /matplotlib -CMD ["python3", "server.py"] -#ENTRYPOINT ["gunicorn", "-k", "geventwebsocket.gunicorn.workers.GeventWebSocketWorker", "-w", "4", "-b", "0.0.0.0:5000", "--timeout", "2000" , "--graceful-timeout", "200", "server:app"] \ No newline at end of file +#CMD ["pypy3", "server.py"] +ENTRYPOINT ["gunicorn", "-k", "geventwebsocket.gunicorn.workers.GeventWebSocketWorker", "-w", "4", "-b", "0.0.0.0:5000", "--timeout", "2000" , "--graceful-timeout", "200", "server:app"] \ No newline at end of file diff --git a/integrator/main.py b/integrator/main.py index 702444c..7b4e7a4 100644 --- a/integrator/main.py +++ b/integrator/main.py @@ -79,7 +79,7 @@ def update_triangle_graph(t: Tree, i, hash, return_start_node=None): t.save_state(i, hash) with catchtime("NEW GRAPH"): - return Tree.max_score_triangle_subgraph( + return t.max_score_triangle_subgraph( t.graph, return_start_node=return_start_node ) diff --git a/integrator/requirements.txt b/integrator/requirements.txt index 89ee445..73ad443 100644 --- a/integrator/requirements.txt +++ b/integrator/requirements.txt @@ -9,11 +9,8 @@ requests cython pyyaml ruamel.yaml -langchain -chromadb -sentence-transformers -tensorboardX -tensorboard + + addict Flask-SocketIO eventlet @@ -24,3 +21,4 @@ gevent-websocket gunicorn redis +networkx \ No newline at end of file diff --git a/integrator/server.py b/integrator/server.py index 629e0a3..f1b973b 100644 --- a/integrator/server.py +++ b/integrator/server.py @@ -128,7 +128,21 @@ def handle_set_state(hash_id): old_state, i = states[hash_id] active_version = Tree.serialize_graph_to_structure( - *Tree.max_score_triangle_subgraph(old_state.graph, return_start_node=True) + *old_state.max_score_triangle_subgraph(old_state.graph, return_start_node=True) + ) + return active_version + + +@socket_event("set_start_node", "set_state") +def set_start_node(new_start_node, hash_id): + print(f"set_start_node {new_start_node} {hash_id}") + + old_state, i = states[hash_id] + old_state.start_node = new_start_node + states[hash_id] = old_state, i + + active_version = Tree.serialize_graph_to_structure( + *old_state.max_score_triangle_subgraph(old_state.graph, return_start_node=True) ) return active_version diff --git a/integrator/setup.py b/integrator/setup.py new file mode 100644 index 0000000..c03088f --- /dev/null +++ b/integrator/setup.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python +from Cython.Build import cythonize +from setuptools import setup + +setup( + ext_modules=cythonize( + "tree.py", compiler_directives={"language_level": "3"}, annotate=True + ) +) diff --git a/integrator/states.py b/integrator/states.py index 2428fc9..cd4d239 100644 --- a/integrator/states.py +++ b/integrator/states.py @@ -2,6 +2,8 @@ import pickle from shutil import rmtree +import regex as re + from integrator.reader import parse_text from integrator.tree import Tree from lib.ls import list_files_with_regex @@ -82,6 +84,20 @@ def __setitem__(self, hash_id, state): tree.save_state(i, hash_id) print(f"saved {i=} {hash_id} {tree=} ") + # Get all state filenames + state_indices = list_files_with_regex( + "states/" + hash_id, r"(?Ptree_state_(?P\d+)\.pkl)" + ) + + # Extract and sort the numeric parts of the filenames + state_indices.sort(reverse=True, key=lambda x: int(x["i"])) + + for state_index in state_indices[3:]: + try: + os.unlink(f"states/{hash_id}/{state_index['filename']}") + except FileNotFoundError: + pass + def __delitem__(self, key): rmtree(self.path(key), ignore_errors=True) try: @@ -95,3 +111,8 @@ def __delitem__(self, key): states = States() + + +if __name__ == "__main__": + v, i = states["1cb10b667a508d3585b760a6ef9e8567b38af5421469a767678007a8a525d911"] + states["1cb10b667a508d3585b760a6ef9e8567b38af5421469a767678007a8a525d911"] = v, i diff --git a/integrator/states/1cb10b667a508d3585b760a6ef9e8567b38af5421469a767678007a8a525d911-meta.pkl b/integrator/states/1cb10b667a508d3585b760a6ef9e8567b38af5421469a767678007a8a525d911-meta.pkl new file mode 100644 index 0000000000000000000000000000000000000000..1446af40abd70a8bf8fd75f4dfbf77c81853ce74 GIT binary patch literal 72 zcmZo*nQF@b0X>!uN%{HN)yXNh~eN$S(p) QWh9ok2ZYpe)lSg^0MCpSxc~qF literal 0 HcmV?d00001 diff --git a/integrator/states/1cb10b667a508d3585b760a6ef9e8567b38af5421469a767678007a8a525d911-text.pkl b/integrator/states/1cb10b667a508d3585b760a6ef9e8567b38af5421469a767678007a8a525d911-text.pkl new file mode 100644 index 0000000000000000000000000000000000000000..d047f47e0162762d1c33fbec75b743d77bd39ca7 GIT binary patch literal 1555 zcmYk6-HzHY5QST*s;WE%Pk>4wf0u{_!3F9H+N!tS!~|FVNo@yhdew($-?zsT2R2t8 z+wpupbH;x@{QLCr9eqFEW3ZUdoHjkP@49hYHJ=@gMjAaNi_Zvp=|v|+9gbXpVsDgbx#koD zWY0afj~T$62hDAZ-C$X3qnN}PG)U=%?&(L+d`Cj2Yo08CgMaF)H9F=2(%KD5ac}W2 zm^S9hoiI8{!L+liRa|@KX$DZ+U%imx#fn}fDWHy~neLr}2$`V|sF5uN70DGt0#lvj zbRsF2f^Mx|L`$qG@ZPAM5}OlB$e15^&<|IuiDwDIgZ8|&G58MpHr;*+ zzGrS#AFs0n-^MV>T}`~pP)8rmx`;ZS0KOA%dfh-j)E|J))<|*TQ%W!fwJnQCK-|7| zOp}Bw0<2~4#4QwK6$E^J6NjpfRTjU;j4Sfk1qmR#h;slv50J??;gS_F*I4tRP6$at zCR?WBZa^L!3`iQS02nl~mQOQ4e4}y(Mu_^ekD_K^#Eo#Yhbk~P7_nYIER_LqhbzG& z8OV0Rjbm_8CdWw&Cc~98;63%h)+0{_x+i{%H_JeG;`h$UfVS4c8E;KDBstK4a*;jN zCkGiwoY7?F9AGDYj{vom4CbKz$pLl}t!M&64ziu>Yu}FyIhf9#wK|4C`f5?KyEAfN z!6_a02&xEBoZgQYG!e)O zkWp$lu%Sy9U^}QlGHsdY!Xq*Q7o?g%QDY3gjR~Fg8V)UicNPdoDM}FT2XriHQ3CL} zr{*hxcVtrXDODMKc28UepEay|OawGk16d+l3AXLc$eyfLj>WWm^g{_MGz;2sj4LwQ zBP5^UEb0XLsAj_>r~(7U*`H?JiKHMz`25AP5C5QMT3+rRb43Ko0#8 z?f>of4Jq5n7CjYcfM^~ZlEayAhV&nw|E0h9O#VK63FdO9W=gAB3{sSpb*8o|G2M2o ztsD9@n$J`WgE&@kw3w*`d=i7eWHy_rG+NG-sY=shKxiDqM7OC*xL)$&&QzmKm7lPh zsm&JFn{7-c`PgiJ?^~O3IE}6_TDqL$Ah4j2=&J4S*6-ebz>>OFd;3ruF0YHD*OYD* z_aiqRxsKMoy`}aL__*KMs?u3g*T&@-@g{L+OSA=+P&Q6AojNKf^XO~%2<@Iu zlEO<76PHHcU~DX_p>!Pe9L9qx@f#mSL{W~NM&Hg6^6-*eI z3}FG2=${@Fy#}=@5BD9vc}YuTH8@Lie@e{p3{2bicj-x-oQN)wY#6m$OXmY8^F)Y(MV%5ep=`QBBJ*k$D&vJ#JBo;4GOu5Y})6pn!@x)*k^Rjro zE#FqI8%Dev!Q{Z{IsF}JCa?B2M>ZrcjRnl1m z$od{5h?oP=_D~WIBwP>zvp5G<&;gpxkCEZ1DBwSf?;rCv&1s`K&{CMHXEcPyo|lEd$6M_m^#SERRrYL$VyGSIaacx|8&q(>}^XXXI z^Qx31NuCz~4#AC$NMbMGM6T`V*ctF;A#ZzNw>TmR*YW~z*_RdZNfLV@L1y}LE0y!8 zw4>n08Wp62I+a>_lwHG}%cT0H0Px8yS^#zEqgWr`SlHkNXPltimU0XA zC$)@?s^I<3iw3vr#$;>!tlZzWh5Z!A5IJUs) zP|!>m7l|G>1gcbnJxDB^a@o#!+nX~UWz^v=5}QE{Wo?lm zhA9N^R8vc!jyuaf!3*KND_;F%_y0OgOTdyIG_GQ;PORTwlya-V`{(GBYMShoZlfgt z+}UA77vMr#wE~-a>pRPv9MTsV~gFscI`0U9ln2c{bv0uBQ@RCn!8e=`N9A3 zN?HQj{jMxzJyNVFqHG25^*i4D*o>5e6!BEM?MK-GA85{U0W9{-*k!C0&tR5UOs(h% z2v-1MKcY9t>pF0l5>&WK(r5+f4YsErB(WF519S!W5gqJZLNQD+S^*YAXV_{G>5Q^C%`d>-<9Zc&cx3EN7M>qYV3LLD13P<`h3|GkP=zI`_(%uK4z`|PI>eD;H ccke4C`rgQKNMbL9=Y(#3CrhSh*PpY$0pV$w-~a#s literal 0 HcmV?d00001 diff --git a/integrator/texts/shell_builtins.txt b/integrator/texts/shell_builtins.txt new file mode 100644 index 0000000..e036b11 --- /dev/null +++ b/integrator/texts/shell_builtins.txt @@ -0,0 +1,61 @@ +1. . - .: . filename [arguments] +2. : - :: : +3. [ - [: [ arg... ] +4. alias - alias: alias [-p] [name[=value] ... ] +5. bg - bg: bg [job_spec ...] +6. bind - bind: bind [-lpsvPSVX] [-m keymap] [-f filename] [-q name] [-u name] [-r keyseq] [-x keyseq:shell-command] [keyseq:readline-function or readline-command] +7. break - break: break [n] +8. builtin - builtin: builtin [shell-builtin [arg ...]] +9. caller - caller: caller [expr] +10. cd - cd: cd [-L|[-P [-e]] [-@]] [dir] +11. command - command: command [-pVv] command [arg ...] +12. compgen - compgen: compgen [-abcdefgjksuv] [-o option] [-A action] [-G globpat] [-W wordlist] [-F function] [-C command] [-X filterpat] [-P prefix] [-S suffix] [word] +13. complete - complete: complete [-abcdefgjksuv] [-pr] [-DEI] [-o option] [-A action] [-G globpat] [-W wordlist] [-F function] [-C command] [-X filterpat] [-P prefix] [-S suffix] [name ...] +14. compopt - compopt: compopt [-o|+o option] [-DEI] [name ...] +15. continue - continue: continue [n] +16. declare - declare: declare [-aAfFgiIlnrtux] [name[=value] ...] or declare -p [-aAfFilnrtux] [name ...] +17. dirs - dirs: dirs [-clpv] [+N] [-N] +18. disown - disown: disown [-h] [-ar] [jobspec ... | pid ...] +19. echo - echo: echo [-neE] [arg ...] +20. enable - enable: enable [-a] [-dnps] [-f filename] [name ...] +21. eval - eval: eval [arg ...] +22. exec - exec: exec [-cl] [-a name] [command [argument ...]] [redirection ...] +23. exit - exit: exit [n] +24. export - export: export [-fn] [name[=value] ...] or export -p +25. false - false: false +26. fc - fc: fc [-e ename] [-lnr] [first] [last] or fc -s [pat=rep] [command] +27. fg - fg: fg [job_spec] +28. getopts - getopts: getopts optstring name [arg ...] +29. hash - hash: hash [-lr] [-p pathname] [-dt] [name ...] +30. help - help: help [-dms] [pattern ...] +31. history - history: history [-c] [-d offset] [n] or history -anrw [filename] or history -ps arg [arg...] +32. jobs - jobs: jobs [-lnprs] [jobspec ...] or jobs -x command [args] +33. kill - kill: kill [-s sigspec | -n signum | -sigspec] pid | jobspec ... or kill -l [sigspec] +34. let - let: let arg [arg ...] +35. local - local: local [option] name[=value] ... +36. logout - logout: logout [n] +37. mapfile - mapfile: mapfile [-d delim] [-n count] [-O origin] [-s count] [-t] [-u fd] [-C callback] [-c quantum] [array] +38. popd - popd: popd [-n] [+N | -N] +39. printf - printf: printf [-v var] format [arguments] +40. pushd - pushd: pushd [-n] [+N | -N | dir] +41. pwd - pwd: pwd [-LP] +42. read - read: read [-ers] [-a array] [-d delim] [-i text] [-n nchars] [-N nchars] [-p prompt] [-t timeout] [-u fd] [name ...] +43. readarray - readarray: readarray [-d delim] [-n count] [-O origin] [-s count] [-t] [-u fd] [-C callback] [-c quantum] [array] +44. readonly - readonly: readonly [-aAf] [name[=value] ...] or readonly -p +45. return - return: return [n] +46. set - set: set [-abefhkmnptuvxBCEHPT] [-o option-name] [--] [-] [arg ...] +47. shift - shift: shift [n] +48. shopt - shopt: shopt [-pqsu] [-o] [optname ...] +49. source - source: source filename [arguments] +50. suspend - suspend: suspend [-f] +51. test - test: test [expr] +52. times - times: times +53. trap - trap: trap [-lp] [[arg] signal_spec ...] +54. true - true: true +55. type - type: type [-afptP] name [name ...] +56. typeset - typeset: typeset [-aAfFgiIlnrtux] name[=value] ... or typeset -p [-aAfFilnrtux] [name ...] +57. ulimit - ulimit: ulimit [-SHabcdefiklmnpqrstuvxPRT] [limit] +58. umask - umask: umask [-p] [-S] [mode] +59. unalias - unalias: unalias [-a] name [name ...] +60. unset - unset: unset [-f] [-v] [-n] [name ...] +61. wait - wait: wait [-fn] [-p var] [id ...] diff --git a/integrator/texts/strange_concepts.txt b/integrator/texts/strange_concepts.txt new file mode 100644 index 0000000..c71ccfc --- /dev/null +++ b/integrator/texts/strange_concepts.txt @@ -0,0 +1,100 @@ +1. Dichotomy +2. Uniformity +3. Diversity +4. Parity +5. Singularity +6. Polarity +7. Conformity +8. Disparity +9. Monotony +10. Multiplicity +11. Homogeneity +12. Heterogeneity +13. Synergy +14. Anomaly +15. Continuity +16. Variance +17. Synchronicity +18. Asymmetry +19. Cohesion +20. Fragmentation +21. Equilibrium +22. Modularity +23. Convergence +24. Divergence +25. Complementarity +26. Ambiguity +27. Uniformity +28. Assimilation +29. Distinction +30. Integration +31. Segregation +32. Invariance +33. Permutation +34. Coalescence +35. Stratification +36. Redundancy +37. Proliferation +38. Contraction +39. Expansion +40. Inclusivity +41. Exclusivity +42. Homogenization +43. Differentiation +44. Symbiosis +45. Antithesis +46. Amalgamation +47. Isolation +48. Interrelation +49. Uniformity +50. Disjunction +51. Consistency +52. Discontinuity +53. Fusion +54. Fission +55. Intersectionality +56. Duplication +57. Deviation +58. Regularity +59. Anisotropy +60. Confluence +61. Dissonance +62. Resonance +63. Elasticity +64. Rigidity +65. Fluctuation +66. Stagnation +67. Transmutation +68. Perpetuity +69. Oscillation +70. Inertia +71. Morphology +72. Homology +73. Reciprocity +74. Reversibility +75. Transcendence +76. Immutability +77. Transformation +78. Reiteration +79. Complexity +80. Simplicity +81. Hybridity +82. Purity +83. Malleability +84. Stability +85. Adaptability +86. Inflexibility +87. Evolution +88. Regression +89. Progression +90. Digression +91. Coexistence +92. Antagonism +93. Harmony +94. Discord +95. Synthesis +96. Decomposition +97. Interdependence +98. Autonomy +99. Modulation +100. Amplification diff --git a/integrator/tree.py b/integrator/tree.py index 1a72859..36dd08c 100644 --- a/integrator/tree.py +++ b/integrator/tree.py @@ -1,3 +1,4 @@ +# cython: language_level=3 import math import os import pickle @@ -19,6 +20,7 @@ def __init__(self, enumerated_texts): self.inputs = enumerated_texts self._populate_graph(enumerated_texts) self._create_sequence_edges() + self.start_node = None def _populate_graph(self, enumerated_texts): for key, text in enumerated_texts: @@ -393,23 +395,25 @@ def top_n_subsuming_nodes(G, n=10): # Return top n nodes return sorted_nodes[:n] - @staticmethod - def max_score_triangle_subgraph(G, return_start_node=False): - _G = G.copy() + def max_score_triangle_subgraph(self, __G, return_start_node=False): + _G = __G.copy() - G = Tree.graph_without_text_sequence(G) + G = Tree.graph_without_text_sequence(_G) try: - depth = math.log(len(G.nodes()), 3) + 1 + depth = math.log(len(G.nodes()), 3) except: depth = 0 - print(f"{depth=}") + # print(f"{depth=}") # Sort nodes by score and get the top 10 - sorted_nodes = Tree.top_n_subsuming_nodes(G, n=4) + if not self.start_node: + sorted_nodes = Tree.top_n_subsuming_nodes(G, n=4) + else: + sorted_nodes = [self.start_node] - print(f"{sorted_nodes=}") + # print(f"{sorted_nodes=}") best_subgraph = None best_start_node = None @@ -456,6 +460,7 @@ def save_state(self, number, hash, filename=None): "graph": self.graph, "yielded": self.yielded, "j": self.j, + "start_node": self.start_node } with open(filename, "wb") as file: pickle.dump(state, file) @@ -487,15 +492,33 @@ def load_state(cls, hash, number=None): ) latest_number = number - with open(filename, "rb") as file: - state = pickle.load(file) - tree = cls([]) - tree.inputs = state["inputs"] - tree.graph = state["graph"] - tree.yielded = state["yielded"] - tree.j = state["j"] + 1 + try: + + with open(filename, "rb") as file: + state = pickle.load(file) + tree = cls([]) + tree.inputs = state["inputs"] + tree.graph = state["graph"] + tree.yielded = state["yielded"] + tree.start_node = state["start_node"] if "start_node" in state else None + tree.j = state["j"] + 1 + + return tree, latest_number + 1 + except MemoryError: + print("Memory error in loading state") + filename = os.path.join( + cls.pickle_folder_path(hash), f"tree_state_{latest_number-1}.pkl" + ) + with open(filename, "rb") as file: + state = pickle.load(file) + tree = cls([]) + tree.inputs = state["inputs"] + tree.graph = state["graph"] + tree.yielded = state["yielded"] + tree.start_node = state["start_node"] if "start_node" in state else None + tree.j = state["j"] + 1 - return tree, latest_number + 1 + return tree, latest_number + 1 def dump_graph(self, hash, graph=None, filename=None): if graph is None: @@ -583,7 +606,7 @@ def test_max_score_triangle_subgraph(): tree.add_relation(n1, n2, relation_type=rel, trident=trident, **attr) # Get the max score path - result_graph = Tree.max_score_triangle_subgraph(tree.graph) + result_graph = tree.max_score_triangle_subgraph(tree.graph) print(result_graph.edges(data=True)) for n1, n2, rel, trident, attr in edges_to_create: @@ -616,7 +639,7 @@ def test_max_score_triangle_subgraph_worse_paths(): tree.add_relation(n1, n2, relation_type=rel, trident=trident, **attr) # Get the max score path - result_graph = Tree.max_score_triangle_subgraph(tree.graph) + result_graph = tree.max_score_triangle_subgraph(tree.graph) print(result_graph.edges(data=True)) for n1, n2, rel, trident, attr in edges_to_create: @@ -714,7 +737,7 @@ def generate_test_data(num_nodes=100, max_score=1.0): def test_with_generated_data(): tree = generate_test_data() - result_graph, start_node = Tree.max_score_triangle_subgraph( + result_graph, start_node = tree.max_score_triangle_subgraph( tree.graph, return_start_node=True ) tree.draw_graph(result_graph, path="generated_graph.png", text_relation=False) diff --git a/linker/Dockerfile b/linker/Dockerfile index 91feb45..b1ba3a9 100644 --- a/linker/Dockerfile +++ b/linker/Dockerfile @@ -1,5 +1,5 @@ # Use an official Python runtime as a parent image -FROM kalilinux/kali-rolling:latest +FROM python:3.11-slim-buster ENV PYTHONUNBUFFERED 1 USER root diff --git a/service/Dockerfile b/service/Dockerfile index b4bac3b..b43a1c8 100644 --- a/service/Dockerfile +++ b/service/Dockerfile @@ -1,5 +1,5 @@ # Use an official Python runtime as a parent image -FROM kalilinux/kali-rolling:latest +FROM python:3.11 ENV PYTHONUNBUFFERED 1 USER root @@ -20,6 +20,7 @@ RUN pip3 config list RUN pip3 install uwsgi RUN --mount=type=cache,target=/root/.cache/pip pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu RUN --mount=type=cache,target=/root/.cache/pip pip3 install --no-cache-dir -r requirements.prod + RUN mkdir /.cache RUN chmod 777 /.cache diff --git a/service/requirements.txt b/service/requirements.txt index b59e054..f3a582c 100644 --- a/service/requirements.txt +++ b/service/requirements.txt @@ -11,4 +11,4 @@ pyyaml farm-haystack[inference] ruamel.yaml langchain -chromadb \ No newline at end of file +chromadb diff --git a/web/src/App.css b/web/src/App.css index 1dbf399..95d45e3 100644 --- a/web/src/App.css +++ b/web/src/App.css @@ -210,7 +210,6 @@ ant-btn { } .button { - background: #333; color: #fff; border: none; width: 30px; diff --git a/web/src/ui/MobileControls.jsx b/web/src/ui/MobileControls.jsx index d7acbbd..b999e5b 100644 --- a/web/src/ui/MobileControls.jsx +++ b/web/src/ui/MobileControls.jsx @@ -122,7 +122,12 @@ export const MobileControls = ({ + + ) +} + +const PuzzleControls = ({ setAction, action, socket, hash }) => { + return ( +
+ +
+ ) +} + +export default PuzzleControls diff --git a/web/src/ui/editor/Editor.jsx b/web/src/ui/editor/Editor.jsx index fd40010..dfbf890 100644 --- a/web/src/ui/editor/Editor.jsx +++ b/web/src/ui/editor/Editor.jsx @@ -50,9 +50,7 @@ export const Editor = () => { const timeoutId = useRef(null) const checkAwaitState = useCallback(() => { - console.log('CHECK ', taskId) if (taskId) { - console.log('CHECK AWAIT STATE', taskId) socket.timeout(3000).emit('patch_poll', taskId) // Clear previous timeout, just in case @@ -139,7 +137,6 @@ export const Editor = () => { active_patching = false setTaskId(null) // Reset taskId after getting result - console.log('RESTART PATCHING', hash, state) socket.timeout(3000).emit('update_state', hash) active_patching = true } @@ -260,7 +257,7 @@ export const Editor = () => { label: 'Puzzle', children: state && ( ), diff --git a/web/src/ui/editor/PuzzleView.jsx b/web/src/ui/editor/PuzzleView.jsx index f12c0f3..f30efa9 100644 --- a/web/src/ui/editor/PuzzleView.jsx +++ b/web/src/ui/editor/PuzzleView.jsx @@ -1,6 +1,41 @@ -import React from 'react' +import React, { useState } from 'react' import { Puzzle } from '../Puzzle' +import PuzzleControls from './Controls' +import { Button } from 'antd' export function PuzzleView(props) { - return + const [action, _setAction] = useState(null) + const setAction = (action) => { + console.log('Setting action to', action) + _setAction(() => action) + } + console.log("ACTION", action) + + + return ( +
+ + + + +
+ ) } diff --git a/worker/Dockerfile b/worker/Dockerfile index 7e8f1ca..132bd71 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-slim +FROM python:3.11-slim-buster WORKDIR /app RUN --mount=type=cache,target=/root/.cache/pip pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu diff --git a/worker/tasks.py b/worker/tasks.py index b8ce08a..dbe05ff 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -19,7 +19,7 @@ def threerarchy(hash_id): print(f"{len(old_state.graph.edges())=}") - old_graph = Tree.max_score_triangle_subgraph( + old_graph = old_state.max_score_triangle_subgraph( old_state.graph, return_start_node=True ) new_graph = update_triangle_graph(old_state, i, hash_id, return_start_node=True)