From c53252af2602a9e1e646a368b2c5882fd52fa013 Mon Sep 17 00:00:00 2001 From: Etherealxx Date: Sun, 10 Sep 2023 07:12:50 +0700 Subject: [PATCH 1/2] sd-webui-reactor as optional extension - Added `sd-webui-reactor` (roop alternative) as optional choosable extension. `additionalextensions.txt` now support running bash code if an extension is selected (mostly for dependencies). --- README.md | 1 + additionalextensions.txt | 7 ++++++- camendurucolab3.py | 12 ++++++++++-- extensioninstaller3.py | 21 ++++++++++++++++++--- volatile_concentration_localux_colab.ipynb | 21 +++++++++++++++++++-- 5 files changed, 54 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 0476e02..5ee8f4b 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Huge thanks to [camenduru](https://github.com/camenduru), without him this colab Read [here](https://github.com/etherealxx/volatile-concentration-localux-colab/blob/main/error403guide.md) for guide to fix it. ### 🆙 Latest Update: +- 10/09/2023 (September): Added `sd-webui-reactor` (roop alternative) as optional choosable extension. `additionalextensions.txt` now support running bash code if an extension is selected (mostly for dependencies). - 12/08/2023 (August): Gradio version bump to v3.37.0 (fixing the bug where extension selection doesn't appear and when orange button is pressed, error JSON input will shows up). ~~gradio_client version bump to v0.2.10 to matches the Gradio version.~~ - 27/07/2023 (July): Memory fix. The sed lines are now synced with camenduru's repo. - 22/07/2023 (July): Added a little bit of documentation on the colab notebook. Removed unused old scripts. Fixed bug where unticking `choose_model` while at the same time ticking `controlnet_models` on the notebook makes SD fails to launch. Now changing branch after running the main cell atleast once will preserve the previously downloaded models and generated outputs. diff --git a/additionalextensions.txt b/additionalextensions.txt index 4ce1792..a655497 100644 --- a/additionalextensions.txt +++ b/additionalextensions.txt @@ -2,6 +2,7 @@ #_You can make a pull request and add your desired extension link here #_Line that starts with #branch will determine the repo's branch, the format is '#branch #_Line that starts with #commit will determine the repo's branch, the format is '#commit +#_Line that starts with #run will be run with subprocess when the extension is selected, the format is '#run ' #@Ahmedkel's request https://github.com/DominikDoom/a1111-sd-webui-tagcomplete @@ -13,4 +14,8 @@ https://github.com/a2569875/stable-diffusion-webui-composable-lora https://github.com/hnmr293/sd-webui-cutoff https://github.com/zanllp/sd-webui-infinite-image-browsing https://github.com/Coyote-A/ultimate-upscale-for-automatic1111 -https://github.com/Bing-su/adetailer \ No newline at end of file +https://github.com/Bing-su/adetailer + +#@otorre1's request, roop alternative +https://github.com/Gourieff/sd-webui-reactor +#run sd-webui-reactor pip install -q insightface==0.7.3 onnx==1.14.0 onnxruntime==1.15.0 opencv-python==4.7.0.72 tqdm \ No newline at end of file diff --git a/camendurucolab3.py b/camendurucolab3.py index 9a81c63..4a190c5 100644 --- a/camendurucolab3.py +++ b/camendurucolab3.py @@ -168,18 +168,26 @@ def rulesbroken(codetoexecute, sedlines=None): # Dumped from the main colab, where it compares every extensions to those chosen by user # ... resulting extensions to be removed/blacklisted extensiontoremove = pickleload(None, 'removedextensions') +extensionrequirements = pickleload(None, 'extrequirements') installextensions = [] for ext_line in extensionlines: + def addlineandcheck(name, line): + installextensions.append(line) + if name in extensionrequirements: + if extensionrequirements[name].startswith(name): + extensionrequirements[name] = extensionrequirements[name].lstrip(name) + installextensions.append(extensionrequirements[name]) + pattern = r"https://github.com/\S+/(\S+)" match = re.search(pattern, ext_line) if match: ext_name = match.group(1) if extensiontoremove: if not ext_name in extensiontoremove: - installextensions.append(ext_line) + addlineandcheck(ext_name, ext_line) else: - installextensions.append(ext_line) + addlineandcheck(ext_name, ext_line) # for x in ("linetoexecute_part1", "linetoexecute_part2", "linetoexecute_part2_1", "linetoexecute_part2_2", "linetoexecute_part3"): # print(f"{x} = {str(eval(x))}") diff --git a/extensioninstaller3.py b/extensioninstaller3.py index fc1f482..6ce4e0c 100644 --- a/extensioninstaller3.py +++ b/extensioninstaller3.py @@ -29,11 +29,13 @@ def list_additional_ext(): with open(addext_txtpath, 'r') as file: lines = [line.rstrip('\n') for line in file] exts = [ext for ext in lines if ext != "" and not ext.startswith("#")] + # commits and branches functions are untested and might not work commits = [ext.lstrip("#commit") for ext in lines if ext != "" and ext.startswith("#commit")] branches = [ext.lstrip("#branch") for ext in lines if ext != "" and ext.startswith("#branch")] - return exts, commits, branches + commands = [ext.lstrip("#run") for ext in lines if ext != "" and ext.startswith("#run")] + return exts, commits, branches, commands -additionalextensions, additionalcommits, additionalbranches = list_additional_ext() +additionalextensions, additionalcommits, additionalbranches, additionalcommands = list_additional_ext() # Dumped from the main colab colaboptions = pickleload(None, 'colaboptions') @@ -69,10 +71,12 @@ def list_additional_ext(): commandtoappend = stripped_line.replace('/content/stable-diffusion-webui', '/content/volatile-concentration-localux') extensionlines.append(commandtoappend) +linestorun = dict() + for addextgithublink in additionalextensions: gitclonestring = 'git clone https://github.com/' repoowner = addextgithublink.split("/")[-2] - reponame = addextgithublink.split("/")[-1] + reponame = addextgithublink.split("/")[-1].strip() for branchline in additionalbranches: if reponame in branchline: specificbranch = branchline.lstrip(reponame).strip() @@ -80,12 +84,23 @@ def list_additional_ext(): break extensionlines.append(f"{gitclonestring}{repoowner}/{reponame} {extensionpath}{reponame}") + for commitline in additionalcommits: if reponame in commitline: specificcommit = commitline.lstrip(reponame).strip() extensionlines.append(f"git checkout {specificcommit} .") break + + for commandline in additionalcommands: + if reponame in commandline: + linestorun[reponame] = commandline.lstrip(reponame).strip() +if linestorun: + pickledump(linestorun, 'extrequirements') +else: + extreqpath = os.path.join(vclvarpath, "extrequirements.pkl") + if os.path.exist(extreqpath): + os.remove(extreqpath) # extensionlines.append(f"{gitclonestring}a2569875/stable-diffusion-webui-composable-lora {extensionpath}stable-diffusion-webui-composable-lora") # extensionlines.append(f"{gitclonestring}DominikDoom/a1111-sd-webui-tagcomplete {extensionpath}a1111-sd-webui-tagcomplete") diff --git a/volatile_concentration_localux_colab.ipynb b/volatile_concentration_localux_colab.ipynb index 9394e32..f7abd03 100644 --- a/volatile_concentration_localux_colab.ipynb +++ b/volatile_concentration_localux_colab.ipynb @@ -18,7 +18,7 @@ "source": [ "##***Project `VCL-colab`***\n", "### All camenduru colab in one spot, synced in realtime\n", - "###### Last time updated: Aug 12, 23\n", + "###### Last time updated: Sep 10, 23\n", "###### (something doesn't work properly? Make sure you use the [latest version](https://colab.research.google.com/github/etherealxx/volatile-concentration-localux-colab/blob/main/volatile_concentration_localux_colab.ipynb), or [report a bug](https://github.com/etherealxx/volatile-concentration-localux-colab/issues).)" ] }, @@ -457,7 +457,7 @@ "!rm -rf /content/volatile-concentration-localux\n", "!rm -rf /content/vcltools\n", "%cd /content\n", - "!git clone -b dev8 https://github.com/etherealxx/volatile-concentration-localux-colab /content/vcltools" + "!git clone -b dev9 https://github.com/etherealxx/volatile-concentration-localux-colab /content/vcltools" ] }, { @@ -498,6 +498,23 @@ "copy_files(folder_A, folder_B)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# to remove folder and it's subdirectories from a specific path. Need this bcoz of the keyword ban\n", + "import os\n", + "keyword = \"reactor\"\n", + "\n", + "extpath = \"/content/volatile-concentration-localux/extensions\"\n", + "for folder in os.listdir(extpath):\n", + " if keyword in folder:\n", + " folderpath = os.path.join(extpath, folder)\n", + " !rm -rf {folderpath}" + ] + }, { "cell_type": "code", "execution_count": null, From 9aeaac055c66f8673c2b7f0736150d4c5f7fea2f Mon Sep 17 00:00:00 2001 From: Etherealxx Date: Sat, 10 Feb 2024 06:42:36 +0700 Subject: [PATCH 2/2] February commit : Gradio version bump readme edit. run colab exclusion. httpx --- README.md | 26 ++-- additionalextensions.txt | 2 +- choosemodel4.py | 137 +++++++++++---------- volatile_concentration_localux_colab.ipynb | 5 +- 4 files changed, 90 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index 5ee8f4b..70e4a92 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,13 @@ [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/etherealxx/volatile-concentration-localux-colab/blob/main/volatile_concentration_localux_colab.ipynb) <- Click here to access the colab # Project VCL-Colab Another camenduru colab ~~clone~~ alternative.😋 +May only works for Colab Pro user. Features: - All camenduru colab flavor in one single colab -- Bypass the damned google colab warning (when it detects `stable-diffusion-webui` and `sd-webui` string) - Option to choose model from a Gradio UI directly on Colab cell output - Automatic update, synced in real time with Camenduru's repo +- ~~Bypass the damned google colab warning (when it detects `stable-diffusion-webui` and `sd-webui` string)~~ MAY NOT WORK ANYMORE. The automatic update works by basically scraping from camenduru's repo, so it will automatically update the model list everytime camenduru's repo got new models.
As long as he doesn't change much on the repo code, this colab will always works without the need to maintain it. @@ -22,12 +23,19 @@ Huge thanks to [camenduru](https://github.com/camenduru), without him this colab Read [here](https://github.com/etherealxx/volatile-concentration-localux-colab/blob/main/error403guide.md) for guide to fix it. ### 🆙 Latest Update: +- 10/01/2024 (February): Gradio version bump to v3.41.2. Updated `choosemodel4.py` to exclude camenduru's 'run' colab. Added `httpx` pip install. Merging the September branch (lol i forgot) - 10/09/2023 (September): Added `sd-webui-reactor` (roop alternative) as optional choosable extension. `additionalextensions.txt` now support running bash code if an extension is selected (mostly for dependencies). -- 12/08/2023 (August): Gradio version bump to v3.37.0 (fixing the bug where extension selection doesn't appear and when orange button is pressed, error JSON input will shows up). ~~gradio_client version bump to v0.2.10 to matches the Gradio version.~~ -- 27/07/2023 (July): Memory fix. The sed lines are now synced with camenduru's repo. -- 22/07/2023 (July): Added a little bit of documentation on the colab notebook. Removed unused old scripts. Fixed bug where unticking `choose_model` while at the same time ticking `controlnet_models` on the notebook makes SD fails to launch. Now changing branch after running the main cell atleast once will preserve the previously downloaded models and generated outputs. -- 20/07/2023 (July): Added functionality for the extension installer where extension's branch and commits are choosable in `additionalextensions.txt`. Removed the whole `libtcmalloc` lines. Adjusted the way this colab gather the code with the recent changes. -- 10/07/2023 (July): Added `sd-webui-cutoff`, `sd-webui-infinite-image-browsing`, `ultimate-upscale-for-automatic1111`, and `adetailer` as optional choosable extension. Now optional extensions are stored on `additionalextensions.txt`. Now optional extensions are listed at the bottom of the extension checkboxes on the gradio UI. -- 07/07/2023 (July): Fixed some typo in the repo extract code (fixed lite branch). Added `torchmetrics==0.11.4` as an additional dependency for lite branch. -- 02/07/2023 (July): Bypass the new colab warning that detects `sd-webui` string. -- 16/06/2023 (June): Added `a1111-sd-webui-tagcomplete` and `composable-lora extension` as optional choosable extension. Fixed 'all extension is missing' bug. + +-
+ Older Updates + + - 12/08/2023 (August): Gradio version bump to v3.37.0 (fixing the bug where extension selection doesn't appear and when orange button is pressed, error JSON input will shows up). ~~gradio_client version bump to v0.2.10 to matches the Gradio version.~~ + - 27/07/2023 (July): Memory fix. The sed lines are now synced with camenduru's repo. + - 22/07/2023 (July): Added a little bit of documentation on the colab notebook. Removed unused old scripts. Fixed bug where unticking `choose_model` while at the same time ticking `controlnet_models` on the notebook makes SD fails to launch. Now changing branch after running the main cell atleast once will preserve the previously downloaded models and generated outputs. + - 20/07/2023 (July): Added functionality for the extension installer where extension's branch and commits are choosable in `additionalextensions.txt`. Removed the whole `libtcmalloc` lines. Adjusted the way this colab gather the code with the recent changes. + - 10/07/2023 (July): Added `sd-webui-cutoff`, `sd-webui-infinite-image-browsing`, `ultimate-upscale-for-automatic1111`, and `adetailer` as optional choosable extension. Now optional extensions are stored on `additionalextensions.txt`. Now optional extensions are listed at the bottom of the extension checkboxes on the gradio UI. + - 07/07/2023 (July): Fixed some typo in the repo extract code (fixed lite branch). Added `torchmetrics==0.11.4` as an additional dependency for lite branch. + - 02/07/2023 (July): Bypass the new colab warning that detects `sd-webui` string. + - 16/06/2023 (June): Added `a1111-sd-webui-tagcomplete` and `composable-lora extension` as optional choosable extension. Fixed 'all extension is missing' bug. +
+ diff --git a/additionalextensions.txt b/additionalextensions.txt index a655497..4323cf0 100644 --- a/additionalextensions.txt +++ b/additionalextensions.txt @@ -18,4 +18,4 @@ https://github.com/Bing-su/adetailer #@otorre1's request, roop alternative https://github.com/Gourieff/sd-webui-reactor -#run sd-webui-reactor pip install -q insightface==0.7.3 onnx==1.14.0 onnxruntime==1.15.0 opencv-python==4.7.0.72 tqdm \ No newline at end of file +#run sd-webui-reactor pip install -q insightface==0.7.3 onnx "onnxruntime-gpu>=1.16.1" opencv-python tqdm \ No newline at end of file diff --git a/choosemodel4.py b/choosemodel4.py index 97e86ce..947a110 100644 --- a/choosemodel4.py +++ b/choosemodel4.py @@ -1,69 +1,70 @@ -import os, math, subprocess, pickle, sys - -branchtype = 'lite' - -if len(sys.argv) == 2: - branchargs = sys.argv[1] - branchtype = branchargs - -import gradio as gr - -# subprocess.run("apt -y install -qq aria2", shell=True, check=True) - -everycolab = f'/content/camendurus/{branchtype}' -everycolabname = [] -colabnamepair = [] -for colabname in os.listdir(everycolab): - colabnamepruned = colabname.partition('_webui_colab.ipynb')[0] - everycolabname.append(colabnamepruned) - -sortedcolabname = sorted(everycolabname) - -vclvarpath = '/content/vclvariables' -def pickledump(vartodump, outputfile): - outputpath = os.path.join(vclvarpath, outputfile + '.pkl') - with open(outputpath, 'wb') as f: - pickle.dump(vartodump, f) - -# 'sortedcolabname' will be accessed by the main colab notebook -pickledump(sortedcolabname, 'sortedcolabname') - - -# totalcolabcount = len(everycolabname) -# for i, colabname in enumerate(sortedcolabname): -# halfall = math.ceil(totalcolabcount / 2) -# numberedname = "{} | {}".format(i, colabname.ljust(30)) -# if i <= halfall: -# colabnamepair.append(numberedname) -# else: -# rev_index = (i - halfall) - 1 -# colabnamepair[rev_index] += "\t" + numberedname - -# for colabpair in colabnamepair: -# print(colabpair) - -# chosencolabname = '' - -# while True: -# choosenumber = input('Choose the number of the model you want: ') -# if choosenumber.isdigit() and int(choosenumber) < totalcolabcount: -# chosencolabname = sortedcolabname[int(choosenumber)] + '_webui_colab.ipynb' -# print("Model from " + chosencolabname + " will be downloaded immediately after all the dependencies is installed. Please wait") -# break -# elif choosenumber == '': -# print("No model will be pre-downloaded. Dependencies installation will continue.") -# break - -# aria2c_lines = [] - -# if chosencolabname: -# if os.path.exists(os.path.join(everycolab, chosencolabname)): -# with open(os.path.join(everycolab, chosencolabname), 'r', encoding='utf-8') as f: -# for line in f: -# stripped_line = line.strip() -# if stripped_line.startswith('"!aria2c'): -# aria2c_lines.append(stripped_line) - -# if aria2c_lines: -# with open('/content/arialist.pkl', 'wb') as f: +import os, math, subprocess, pickle, sys + +branchtype = 'lite' + +if len(sys.argv) == 2: + branchargs = sys.argv[1] + branchtype = branchargs + +import gradio as gr + +# subprocess.run("apt -y install -qq aria2", shell=True, check=True) + +everycolab = f'/content/camendurus/{branchtype}' +everycolabname = [] +colabnamepair = [] +for colabname in os.listdir(everycolab): + if not colabname.endswith('_run.ipynb'): + colabnamepruned = colabname.partition('_webui_colab.ipynb')[0] + everycolabname.append(colabnamepruned) + +sortedcolabname = sorted(everycolabname) + +vclvarpath = '/content/vclvariables' +def pickledump(vartodump, outputfile): + outputpath = os.path.join(vclvarpath, outputfile + '.pkl') + with open(outputpath, 'wb') as f: + pickle.dump(vartodump, f) + +# 'sortedcolabname' will be accessed by the main colab notebook +pickledump(sortedcolabname, 'sortedcolabname') + + +# totalcolabcount = len(everycolabname) +# for i, colabname in enumerate(sortedcolabname): +# halfall = math.ceil(totalcolabcount / 2) +# numberedname = "{} | {}".format(i, colabname.ljust(30)) +# if i <= halfall: +# colabnamepair.append(numberedname) +# else: +# rev_index = (i - halfall) - 1 +# colabnamepair[rev_index] += "\t" + numberedname + +# for colabpair in colabnamepair: +# print(colabpair) + +# chosencolabname = '' + +# while True: +# choosenumber = input('Choose the number of the model you want: ') +# if choosenumber.isdigit() and int(choosenumber) < totalcolabcount: +# chosencolabname = sortedcolabname[int(choosenumber)] + '_webui_colab.ipynb' +# print("Model from " + chosencolabname + " will be downloaded immediately after all the dependencies is installed. Please wait") +# break +# elif choosenumber == '': +# print("No model will be pre-downloaded. Dependencies installation will continue.") +# break + +# aria2c_lines = [] + +# if chosencolabname: +# if os.path.exists(os.path.join(everycolab, chosencolabname)): +# with open(os.path.join(everycolab, chosencolabname), 'r', encoding='utf-8') as f: +# for line in f: +# stripped_line = line.strip() +# if stripped_line.startswith('"!aria2c'): +# aria2c_lines.append(stripped_line) + +# if aria2c_lines: +# with open('/content/arialist.pkl', 'wb') as f: # pickle.dump(aria2c_lines, f) \ No newline at end of file diff --git a/volatile_concentration_localux_colab.ipynb b/volatile_concentration_localux_colab.ipynb index f7abd03..7b289d2 100644 --- a/volatile_concentration_localux_colab.ipynb +++ b/volatile_concentration_localux_colab.ipynb @@ -18,7 +18,7 @@ "source": [ "##***Project `VCL-colab`***\n", "### All camenduru colab in one spot, synced in realtime\n", - "###### Last time updated: Sep 10, 23\n", + "###### Last time updated: Feb 10, 24\n", "###### (something doesn't work properly? Make sure you use the [latest version](https://colab.research.google.com/github/etherealxx/volatile-concentration-localux-colab/blob/main/volatile_concentration_localux_colab.ipynb), or [report a bug](https://github.com/etherealxx/volatile-concentration-localux-colab/issues).)" ] }, @@ -82,7 +82,8 @@ "\n", "%env TF_CPP_MIN_LOG_LEVEL=1\n", "!git clone https://github.com/etherealxx/volatile-concentration-localux-colab /content/vcltools\n", - "!pip install -q gradio==3.37.0\n", + "!pip install -q gradio==3.41.2\n", + "!pip install -q pastebin-replace httpx==0.24.1\n", "\n", "emptymodel = False\n", "\n",