cc

Makefile

@@ -1,4 +1,8 @@
 -include .env
+ifneq ($(ENV_FILE),)
+	include $(ENV_FILE)
+	export
+endif
 
 export COMPOSE_PROFILES=default
 
@@ -14,6 +18,25 @@ start:
 	docker compose up -d
 
 
+build:
+	eval $(minikube docker-env)  &&docker compose down -v
+	BUILDKIT_PROGRESS=plain docker compose build
+
+kompose:
+	$(MAKE) ENV_FILE=.env _kompose
+_kompose:
+	kompose convert -f compose.yml -o k8s/
+
+apply:
+	eval $$(minikube docker-env) && kubectl apply -f k8s/
+
+minikube-add-images:
+	eval $$(minikube docker-env) && docker build -t worker:latest worker/.
+	eval $$(minikube docker-env) && docker build -t queue:latest worker/.
+	eval $$(minikube docker-env) && docker build -t web:latest web/.
+	eval $$(minikube docker-env) && docker build -t classifier:latest classifier/.
+	eval $$(minikube docker-env) && docker build -t integrator:latest integrator/.
+
 stop:
 	docker compose down -v
 

classifier/Dockerfile

@@ -1,5 +1,5 @@
 # Use an official Python runtime as a parent image
-FROM kalilinux/kali-rolling:latest
+FROM python:3.11-slim-buster
 
 ENV PYTHONUNBUFFERED 1
 USER root

compose.yml

@@ -1,6 +1,7 @@
 version: "3"
 services:
   nginx:
+
     image: nginx:latest
     container_name: nginx
     restart: unless-stopped
@@ -27,6 +28,7 @@ services:
       - ./.certs:$CERTS
 
   server:
+    image: server
 
     logging:
       driver: "json-file"
@@ -57,6 +59,7 @@ services:
       - "./lib:/app/lib"
 
   service:
+    image: service
 
     logging:
       driver: "json-file"
@@ -91,6 +94,7 @@ services:
       - "./.caches/.cache-cr:/chroma:rw"
       - "./.caches/.cache-hf:/.cache:rw"
   integrator:
+    image: integrator:latest
 
     logging:
       driver: "json-file"
@@ -107,7 +111,6 @@ services:
         SYSTEM: /system
 
     restart: unless-stopped
-    image: integrator
 
     container_name: integrator
     env_file:
@@ -137,6 +140,7 @@ services:
       - "./.caches/.cache-mpl:/.config:rw"
 
   linker:
+    image: linker:latest
     profiles:
       - linker
     mem_limit: 10g
@@ -174,6 +178,8 @@ services:
 
 
   classifier:
+    image: classifier
+
     profiles:
       - classifier
     mem_limit: 10g
@@ -210,7 +216,7 @@ services:
 
 
   web:
-
+    image: web:latest
     logging:
       driver: "json-file"
       options:
@@ -261,6 +267,7 @@ services:
         - "./.caches/.cache-mpl:/.config:rw"
 
   queue:
+    image: queue:latest
     build: worker
     command: ["python", "app.py"]
     ports:

integrator/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11-slim-buster
+FROM pypy:latest
 
 ENV PYTHONUNBUFFERED 1
 USER root
@@ -12,8 +12,10 @@ RUN apt install -y gcc python3-pip
 RUN pip3 install --upgrade pip
 RUN pip3 config list
 RUN pip3 install uwsgi
-RUN --mount=type=cache,target=/root/.cache/pip  pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
+#RUN --mount=type=cache,target=/root/.cache/pip  pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
 COPY requirements.txt /app/
+RUN apt install libatlas-base-dev -y
+RUN apt install gfortran -y
 RUN --mount=type=cache,target=/root/.cache/pip  pip3 install --no-cache-dir -r requirements.txt
 RUN mkdir /.cache
 RUN chmod 777 /.cache
@@ -24,5 +26,5 @@ ENV FLASK_ENV=development
 ENV MPLCONFIGDIR=/config/matplotlib
 RUN mkdir /matplotlib
 RUN chmod 777 /matplotlib
-CMD ["python3", "server.py"]
-#ENTRYPOINT ["gunicorn", "-k", "geventwebsocket.gunicorn.workers.GeventWebSocketWorker", "-w", "4", "-b", "0.0.0.0:5000", "--timeout", "2000" , "--graceful-timeout", "200", "server:app"]
+#CMD ["pypy3", "server.py"]
+ENTRYPOINT ["gunicorn", "-k", "geventwebsocket.gunicorn.workers.GeventWebSocketWorker", "-w", "4", "-b", "0.0.0.0:5000", "--timeout", "2000" , "--graceful-timeout", "200", "server:app"]

integrator/main.py

@@ -79,7 +79,7 @@ def update_triangle_graph(t: Tree, i, hash, return_start_node=None):
             t.save_state(i, hash)
 
     with catchtime("NEW GRAPH"):
-        return Tree.max_score_triangle_subgraph(
+        return t.max_score_triangle_subgraph(
             t.graph, return_start_node=return_start_node
         )
 

integrator/requirements.txt

@@ -9,11 +9,8 @@ requests
 cython
 pyyaml
 ruamel.yaml
-langchain
-chromadb
-sentence-transformers
-tensorboardX
-tensorboard
+
+
 addict
 Flask-SocketIO
 eventlet
@@ -24,3 +21,4 @@ gevent-websocket
 gunicorn
 redis
 
+networkx

integrator/server.py

@@ -128,7 +128,21 @@ def handle_set_state(hash_id):
     old_state, i = states[hash_id]
 
     active_version = Tree.serialize_graph_to_structure(
-        *Tree.max_score_triangle_subgraph(old_state.graph, return_start_node=True)
+        *old_state.max_score_triangle_subgraph(old_state.graph, return_start_node=True)
+    )
+    return active_version
+
+
+@socket_event("set_start_node", "set_state")
+def set_start_node(new_start_node, hash_id):
+    print(f"set_start_node {new_start_node} {hash_id}")
+
+    old_state, i = states[hash_id]
+    old_state.start_node = new_start_node
+    states[hash_id] = old_state, i
+
+    active_version = Tree.serialize_graph_to_structure(
+        *old_state.max_score_triangle_subgraph(old_state.graph, return_start_node=True)
     )
     return active_version
 

integrator/setup.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+from Cython.Build import cythonize
+from setuptools import setup
+
+setup(
+    ext_modules=cythonize(
+        "tree.py", compiler_directives={"language_level": "3"}, annotate=True
+    )
+)

integrator/states.py

@@ -2,6 +2,8 @@
 import pickle
 from shutil import rmtree
 
+import regex as re
+
 from integrator.reader import parse_text
 from integrator.tree import Tree
 from lib.ls import list_files_with_regex
@@ -82,6 +84,20 @@ def __setitem__(self, hash_id, state):
             tree.save_state(i, hash_id)
             print(f"saved {i=} {hash_id} {tree=} ")
 
+            # Get all state filenames
+            state_indices = list_files_with_regex(
+                "states/" + hash_id, r"(?P<filename>tree_state_(?P<i>\d+)\.pkl)"
+            )
+
+            # Extract and sort the numeric parts of the filenames
+            state_indices.sort(reverse=True, key=lambda x: int(x["i"]))
+
+            for state_index in state_indices[3:]:
+                try:
+                    os.unlink(f"states/{hash_id}/{state_index['filename']}")
+                except FileNotFoundError:
+                    pass
+
     def __delitem__(self, key):
         rmtree(self.path(key), ignore_errors=True)
         try:
@@ -95,3 +111,8 @@ def __delitem__(self, key):
 
 
 states = States()
+
+
+if __name__ == "__main__":
+    v, i = states["1cb10b667a508d3585b760a6ef9e8567b38af5421469a767678007a8a525d911"]
+    states["1cb10b667a508d3585b760a6ef9e8567b38af5421469a767678007a8a525d911"] = v, i

integrator/texts/shell_builtins.txt

@@ -0,0 +1,61 @@
+1. . - .: . filename [arguments]
+2. : - :: :
+3. [ - [: [ arg... ]
+4. alias - alias: alias [-p] [name[=value] ... ]
+5. bg - bg: bg [job_spec ...]
+6. bind - bind: bind [-lpsvPSVX] [-m keymap] [-f filename] [-q name] [-u name] [-r keyseq] [-x keyseq:shell-command] [keyseq:readline-function or readline-command]
+7. break - break: break [n]
+8. builtin - builtin: builtin [shell-builtin [arg ...]]
+9. caller - caller: caller [expr]
+10. cd - cd: cd [-L|[-P [-e]] [-@]] [dir]
+11. command - command: command [-pVv] command [arg ...]
+12. compgen - compgen: compgen [-abcdefgjksuv] [-o option] [-A action] [-G globpat] [-W wordlist] [-F function] [-C command] [-X filterpat] [-P prefix] [-S suffix] [word]
+13. complete - complete: complete [-abcdefgjksuv] [-pr] [-DEI] [-o option] [-A action] [-G globpat] [-W wordlist] [-F function] [-C command] [-X filterpat] [-P prefix] [-S suffix] [name ...]
+14. compopt - compopt: compopt [-o|+o option] [-DEI] [name ...]
+15. continue - continue: continue [n]
+16. declare - declare: declare [-aAfFgiIlnrtux] [name[=value] ...] or declare -p [-aAfFilnrtux] [name ...]
+17. dirs - dirs: dirs [-clpv] [+N] [-N]
+18. disown - disown: disown [-h] [-ar] [jobspec ... | pid ...]
+19. echo - echo: echo [-neE] [arg ...]
+20. enable - enable: enable [-a] [-dnps] [-f filename] [name ...]
+21. eval - eval: eval [arg ...]
+22. exec - exec: exec [-cl] [-a name] [command [argument ...]] [redirection ...]
+23. exit - exit: exit [n]
+24. export - export: export [-fn] [name[=value] ...] or export -p
+25. false - false: false
+26. fc - fc: fc [-e ename] [-lnr] [first] [last] or fc -s [pat=rep] [command]
+27. fg - fg: fg [job_spec]
+28. getopts - getopts: getopts optstring name [arg ...]
+29. hash - hash: hash [-lr] [-p pathname] [-dt] [name ...]
+30. help - help: help [-dms] [pattern ...]
+31. history - history: history [-c] [-d offset] [n] or history -anrw [filename] or history -ps arg [arg...]
+32. jobs - jobs: jobs [-lnprs] [jobspec ...] or jobs -x command [args]
+33. kill - kill: kill [-s sigspec | -n signum | -sigspec] pid | jobspec ... or kill -l [sigspec]
+34. let - let: let arg [arg ...]
+35. local - local: local [option] name[=value] ...
+36. logout - logout: logout [n]
+37. mapfile - mapfile: mapfile [-d delim] [-n count] [-O origin] [-s count] [-t] [-u fd] [-C callback] [-c quantum] [array]
+38. popd - popd: popd [-n] [+N | -N]
+39. printf - printf: printf [-v var] format [arguments]
+40. pushd - pushd: pushd [-n] [+N | -N | dir]
+41. pwd - pwd: pwd [-LP]
+42. read - read: read [-ers] [-a array] [-d delim] [-i text] [-n nchars] [-N nchars] [-p prompt] [-t timeout] [-u fd] [name ...]
+43. readarray - readarray: readarray [-d delim] [-n count] [-O origin] [-s count] [-t] [-u fd] [-C callback] [-c quantum] [array]
+44. readonly - readonly: readonly [-aAf] [name[=value] ...] or readonly -p
+45. return - return: return [n]
+46. set - set: set [-abefhkmnptuvxBCEHPT] [-o option-name] [--] [-] [arg ...]
+47. shift - shift: shift [n]
+48. shopt - shopt: shopt [-pqsu] [-o] [optname ...]
+49. source - source: source filename [arguments]
+50. suspend - suspend: suspend [-f]
+51. test - test: test [expr]
+52. times - times: times
+53. trap - trap: trap [-lp] [[arg] signal_spec ...]
+54. true - true: true
+55. type - type: type [-afptP] name [name ...]
+56. typeset - typeset: typeset [-aAfFgiIlnrtux] name[=value] ... or typeset -p [-aAfFilnrtux] [name ...]
+57. ulimit - ulimit: ulimit [-SHabcdefiklmnpqrstuvxPRT] [limit]
+58. umask - umask: umask [-p] [-S] [mode]
+59. unalias - unalias: unalias [-a] name [name ...]
+60. unset - unset: unset [-f] [-v] [-n] [name ...]
+61. wait - wait: wait [-fn] [-p var] [id ...]