stablediffusion_dreambooth_fineturing3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0639cab2-e673-49c3-bac9-4e9627747172",
   "metadata": {},
   "outputs": [],
   "source": [
    "!git clone https://github.com/xieyongliang/stable-diffusion-webui.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "8314fc9b-c468-497b-abcc-259ec792154c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "import boto3\n",
    "from sagemaker.pytorch import PyTorch\n",
    "sagemaker_session = sagemaker.Session()\n",
    "bucket = sagemaker_session.default_bucket()\n",
    "role = sagemaker.get_execution_role()\n",
    "account_id = boto3.client('sts').get_caller_identity().get('Account')\n",
    "region_name = boto3.session.Session().region_name\n",
    "images_s3uri = 's3://{0}/dreambooth/images/'.format(bucket)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "516418fb-5755-4e40-b0df-ca80c085067e",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2023-02-27 09:27:26--  https://d1xkebsgyt7kzd.cloudfront.net/R_1.jpg\n",
      "Resolving d1xkebsgyt7kzd.cloudfront.net (d1xkebsgyt7kzd.cloudfront.net)... 13.33.100.75, 13.33.100.45, 13.33.100.108, ...\n",
      "Connecting to d1xkebsgyt7kzd.cloudfront.net (d1xkebsgyt7kzd.cloudfront.net)|13.33.100.75|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 38281 (37K) [image/jpeg]\n",
      "Saving to: ‘R_1.jpg’\n",
      "\n",
      "R_1.jpg             100%[===================>]  37.38K  --.-KB/s    in 0s      \n",
      "\n",
      "2023-02-27 09:27:26 (306 MB/s) - ‘R_1.jpg’ saved [38281/38281]\n",
      "\n",
      "--2023-02-27 09:27:27--  https://d1xkebsgyt7kzd.cloudfront.net/R_2.jpg\n",
      "Resolving d1xkebsgyt7kzd.cloudfront.net (d1xkebsgyt7kzd.cloudfront.net)... 13.33.100.121, 13.33.100.108, 13.33.100.45, ...\n",
      "Connecting to d1xkebsgyt7kzd.cloudfront.net (d1xkebsgyt7kzd.cloudfront.net)|13.33.100.121|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 33919 (33K) [image/jpeg]\n",
      "Saving to: ‘R_2.jpg’\n",
      "\n",
      "R_2.jpg             100%[===================>]  33.12K  --.-KB/s    in 0s      \n",
      "\n",
      "2023-02-27 09:27:27 (266 MB/s) - ‘R_2.jpg’ saved [33919/33919]\n",
      "\n",
      "--2023-02-27 09:27:27--  https://d1xkebsgyt7kzd.cloudfront.net/R_3.jpg\n",
      "Resolving d1xkebsgyt7kzd.cloudfront.net (d1xkebsgyt7kzd.cloudfront.net)... 13.33.100.75, 13.33.100.121, 13.33.100.108, ...\n",
      "Connecting to d1xkebsgyt7kzd.cloudfront.net (d1xkebsgyt7kzd.cloudfront.net)|13.33.100.75|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 118319 (116K) [image/jpeg]\n",
      "Saving to: ‘R_3.jpg’\n",
      "\n",
      "R_3.jpg             100%[===================>] 115.55K  --.-KB/s    in 0s      \n",
      "\n",
      "2023-02-27 09:27:27 (277 MB/s) - ‘R_3.jpg’ saved [118319/118319]\n",
      "\n",
      "--2023-02-27 09:27:27--  https://d1xkebsgyt7kzd.cloudfront.net/R_4.jpg\n",
      "Resolving d1xkebsgyt7kzd.cloudfront.net (d1xkebsgyt7kzd.cloudfront.net)... 13.33.100.45, 13.33.100.75, 13.33.100.121, ...\n",
      "Connecting to d1xkebsgyt7kzd.cloudfront.net (d1xkebsgyt7kzd.cloudfront.net)|13.33.100.45|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 88012 (86K) [image/jpeg]\n",
      "Saving to: ‘R_4.jpg’\n",
      "\n",
      "R_4.jpg             100%[===================>]  85.95K  --.-KB/s    in 0.005s  \n",
      "\n",
      "2023-02-27 09:27:27 (15.9 MB/s) - ‘R_4.jpg’ saved [88012/88012]\n",
      "\n",
      "--2023-02-27 09:27:27--  https://d1xkebsgyt7kzd.cloudfront.net/R_5.jpg\n",
      "Resolving d1xkebsgyt7kzd.cloudfront.net (d1xkebsgyt7kzd.cloudfront.net)... 13.33.100.108, 13.33.100.45, 13.33.100.75, ...\n",
      "Connecting to d1xkebsgyt7kzd.cloudfront.net (d1xkebsgyt7kzd.cloudfront.net)|13.33.100.108|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 334713 (327K) [image/jpeg]\n",
      "Saving to: ‘R_5.jpg’\n",
      "\n",
      "R_5.jpg             100%[===================>] 326.87K  --.-KB/s    in 0.003s  \n",
      "\n",
      "2023-02-27 09:27:28 (113 MB/s) - ‘R_5.jpg’ saved [334713/334713]\n",
      "\n",
      "upload: images/R_4.jpg to s3://sagemaker-ap-southeast-1-687912291502/dreambooth/images/R_4.jpg\n",
      "upload: images/R_3.jpg to s3://sagemaker-ap-southeast-1-687912291502/dreambooth/images/R_3.jpg\n",
      "upload: images/R_5.jpg to s3://sagemaker-ap-southeast-1-687912291502/dreambooth/images/R_5.jpg\n",
      "upload: images/R_1.jpg to s3://sagemaker-ap-southeast-1-687912291502/dreambooth/images/R_1.jpg\n",
      "upload: images/R_2.jpg to s3://sagemaker-ap-southeast-1-687912291502/dreambooth/images/R_2.jpg\n"
     ]
    }
   ],
   "source": [
    "imgs=\"https://d1xkebsgyt7kzd.cloudfront.net/R_1.jpg,https://d1xkebsgyt7kzd.cloudfront.net/R_2.jpg,https://d1xkebsgyt7kzd.cloudfront.net/R_3.jpg,https://d1xkebsgyt7kzd.cloudfront.net/R_4.jpg,https://d1xkebsgyt7kzd.cloudfront.net/R_5.jpg\"\n",
    "for image in imgs.split(\",\"):\n",
    "    !wget $image\n",
    "!mv ./*.jpg ./images/\n",
    "!aws s3 cp images $images_s3uri --recursive    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "09ff0467-9f0a-4c42-8af4-7b3db7f7ccc8",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n1:model_name: str = \"\",\\n2:adam_beta1: float = 0.9,\\n3:adam_beta2: float = 0.999,\\n4:adam_epsilon: float = 1e-8,\\n5:adam_weight_decay: float = 0.01,\\n6:attention: str = \"default\",\\n7:center_crop: bool = True,\\n8:concepts_path: str = \"\",\\n9:custom_model_name: str = \"\",\\n10:epoch_pause_frequency: int = 0,\\n11:epoch_pause_time: int = 0,\\n12:gradient_accumulation_steps: int = 1,\\n13:gradient_checkpointing: bool = True,\\n14:half_model: bool = False,\\n15:has_ema: bool = False,\\n16:hflip: bool = False,\\n17:learning_rate: float = 0.00000172,\\n18:lora_learning_rate: float = 1e-4,\\n19:lora_txt_learning_rate: float = 5e-5,\\n20:lr_scheduler: str = \\'constant\\',\\n21:lr_warmup_steps: int = 0,\\n22:max_token_length: int = 75,\\n23:max_train_steps: int = 1000,\\n24:mixed_precision: str = \"fp16\",\\n25:model_path: str = \"\",\\n26:not_cache_latents=False,\\n27:num_train_epochs: int = 1,\\n28:pad_tokens: bool = True,\\n29:pretrained_vae_name_or_path: str = \"\",\\n30:prior_loss_weight: float = 1.0,\\n31:resolution: int = 512,\\n32:revision: int = 0,\\n33:sample_batch_size: int = 1,\\n34:save_class_txt: bool = False,\\n35:save_embedding_every: int = 500,\\n36:save_preview_every: int = 500,\\n37:save_use_global_counts: bool = False,\\n38:save_use_epochs: bool = False,\\n39:scale_lr: bool = False,\\n40:scheduler: str = \"ddim\",\\n41:src: str = \"\",\\n42:shuffle_tags: bool = False,\\n43:train_batch_size: int = 1,\\n44:train_text_encoder: bool = True,\\n45:use_8bit_adam: bool = True,\\n46:use_concepts: bool = False,\\n47:use_cpu: bool = False,\\n48:use_ema: bool = True,\\n49:use_lora: bool = False,\\n50:v2: bool = False,\\n51:c1_class_data_dir: str = \"\",\\n52:c1_class_guidance_scale: float = 7.5,\\n53:c1_class_infer_steps: int = 60,\\n54:c1_class_negative_prompt: str = \"\",\\n55:c1_class_prompt: str = \"\",\\n56:c1_class_token: str = \"\",\\n57:c1_instance_data_dir: str = \"\",\\n58:c1_instance_prompt: str = \"\",\\n59:c1_instance_token: str = \"\",\\n60:c1_max_steps: int = -1,\\n61:c1_n_save_sample: int = 1,\\n62:c1_num_class_images: int = 0,\\n63:c1_sample_seed: int = -1,\\n64:c1_save_guidance_scale: float = 7.5,\\n65:c1_save_infer_steps: int = 60,\\n66:c1_save_sample_negative_prompt: str = \"\",\\n67:c1_save_sample_prompt: str = \"\",\\n68:c1_save_sample_template: str = \"\",\\n69:c2_class_data_dir: str = \"\",\\n70:c2_class_guidance_scale: float = 7.5,\\n71:c2_class_infer_steps: int = 60,\\n72:c2_class_negative_prompt: str = \"\",\\n73:c2_class_prompt: str = \"\",\\n74:c2_class_token: str = \"\",\\n75:c2_instance_data_dir: str = \"\",\\n76:c2_instance_prompt: str = \"\",\\n77:c2_instance_token: str = \"\",\\n78:c2_max_steps: int = -1,\\n79:c2_n_save_sample: int = 1,\\n80:c2_num_class_images: int = 0,\\n81:c2_sample_seed: int = -1,\\n82:c2_save_guidance_scale: float = 7.5,\\n83:c2_save_infer_steps: int = 60,\\n84:c2_save_sample_negative_prompt: str = \"\",\\n85:c2_save_sample_prompt: str = \"\",\\n86:c2_save_sample_template: str = \"\",\\n87:c3_class_data_dir: str = \"\",\\n88:c3_class_guidance_scale: float = 7.5,\\n89:c3_class_infer_steps: int = 60,\\n90:c3_class_negative_prompt: str = \"\",\\n91:c3_class_prompt: str = \"\",\\n92:c3_class_token: str = \"\",\\n93:c3_instance_data_dir: str = \"\",\\n94:c3_instance_prompt: str = \"\",\\n95:c3_instance_token: str = \"\",\\n96:c3_max_steps: int = -1,\\n97:c3_n_save_sample: int = 1,\\n98:c3_num_class_images: int = 0,\\n99:c3_sample_seed: int = -1,\\n100:c3_save_guidance_scale: float = 7.5,\\n101:c3_save_infer_steps: int = 60,\\n102:c3_save_sample_negative_prompt: str = \"\",\\n103:c3_save_sample_prompt: str = \"\",\\n104:c3_save_sample_template: str = \"\",\\n105:concepts_list=None\\n'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "1:model_name: str = \"\",\n",
    "2:adam_beta1: float = 0.9,\n",
    "3:adam_beta2: float = 0.999,\n",
    "4:adam_epsilon: float = 1e-8,\n",
    "5:adam_weight_decay: float = 0.01,\n",
    "6:attention: str = \"default\",\n",
    "7:center_crop: bool = True,\n",
    "8:concepts_path: str = \"\",\n",
    "9:custom_model_name: str = \"\",\n",
    "10:epoch_pause_frequency: int = 0,\n",
    "11:epoch_pause_time: int = 0,\n",
    "12:gradient_accumulation_steps: int = 1,\n",
    "13:gradient_checkpointing: bool = True,\n",
    "14:half_model: bool = False,\n",
    "15:has_ema: bool = False,\n",
    "16:hflip: bool = False,\n",
    "17:learning_rate: float = 0.00000172,\n",
    "18:lora_learning_rate: float = 1e-4,\n",
    "19:lora_txt_learning_rate: float = 5e-5,\n",
    "20:lr_scheduler: str = 'constant',\n",
    "21:lr_warmup_steps: int = 0,\n",
    "22:max_token_length: int = 75,\n",
    "23:max_train_steps: int = 1000,\n",
    "24:mixed_precision: str = \"fp16\",\n",
    "25:model_path: str = \"\",\n",
    "26:not_cache_latents=False,\n",
    "27:num_train_epochs: int = 1,\n",
    "28:pad_tokens: bool = True,\n",
    "29:pretrained_vae_name_or_path: str = \"\",\n",
    "30:prior_loss_weight: float = 1.0,\n",
    "31:resolution: int = 512,\n",
    "32:revision: int = 0,\n",
    "33:sample_batch_size: int = 1,\n",
    "34:save_class_txt: bool = False,\n",
    "35:save_embedding_every: int = 500,\n",
    "36:save_preview_every: int = 500,\n",
    "37:save_use_global_counts: bool = False,\n",
    "38:save_use_epochs: bool = False,\n",
    "39:scale_lr: bool = False,\n",
    "40:scheduler: str = \"ddim\",\n",
    "41:src: str = \"\",\n",
    "42:shuffle_tags: bool = False,\n",
    "43:train_batch_size: int = 1,\n",
    "44:train_text_encoder: bool = True,\n",
    "45:use_8bit_adam: bool = True,\n",
    "46:use_concepts: bool = False,\n",
    "47:use_cpu: bool = False,\n",
    "48:use_ema: bool = True,\n",
    "49:use_lora: bool = False,\n",
    "50:v2: bool = False,\n",
    "51:c1_class_data_dir: str = \"\",\n",
    "52:c1_class_guidance_scale: float = 7.5,\n",
    "53:c1_class_infer_steps: int = 60,\n",
    "54:c1_class_negative_prompt: str = \"\",\n",
    "55:c1_class_prompt: str = \"\",\n",
    "56:c1_class_token: str = \"\",\n",
    "57:c1_instance_data_dir: str = \"\",\n",
    "58:c1_instance_prompt: str = \"\",\n",
    "59:c1_instance_token: str = \"\",\n",
    "60:c1_max_steps: int = -1,\n",
    "61:c1_n_save_sample: int = 1,\n",
    "62:c1_num_class_images: int = 0,\n",
    "63:c1_sample_seed: int = -1,\n",
    "64:c1_save_guidance_scale: float = 7.5,\n",
    "65:c1_save_infer_steps: int = 60,\n",
    "66:c1_save_sample_negative_prompt: str = \"\",\n",
    "67:c1_save_sample_prompt: str = \"\",\n",
    "68:c1_save_sample_template: str = \"\",\n",
    "69:c2_class_data_dir: str = \"\",\n",
    "70:c2_class_guidance_scale: float = 7.5,\n",
    "71:c2_class_infer_steps: int = 60,\n",
    "72:c2_class_negative_prompt: str = \"\",\n",
    "73:c2_class_prompt: str = \"\",\n",
    "74:c2_class_token: str = \"\",\n",
    "75:c2_instance_data_dir: str = \"\",\n",
    "76:c2_instance_prompt: str = \"\",\n",
    "77:c2_instance_token: str = \"\",\n",
    "78:c2_max_steps: int = -1,\n",
    "79:c2_n_save_sample: int = 1,\n",
    "80:c2_num_class_images: int = 0,\n",
    "81:c2_sample_seed: int = -1,\n",
    "82:c2_save_guidance_scale: float = 7.5,\n",
    "83:c2_save_infer_steps: int = 60,\n",
    "84:c2_save_sample_negative_prompt: str = \"\",\n",
    "85:c2_save_sample_prompt: str = \"\",\n",
    "86:c2_save_sample_template: str = \"\",\n",
    "87:c3_class_data_dir: str = \"\",\n",
    "88:c3_class_guidance_scale: float = 7.5,\n",
    "89:c3_class_infer_steps: int = 60,\n",
    "90:c3_class_negative_prompt: str = \"\",\n",
    "91:c3_class_prompt: str = \"\",\n",
    "92:c3_class_token: str = \"\",\n",
    "93:c3_instance_data_dir: str = \"\",\n",
    "94:c3_instance_prompt: str = \"\",\n",
    "95:c3_instance_token: str = \"\",\n",
    "96:c3_max_steps: int = -1,\n",
    "97:c3_n_save_sample: int = 1,\n",
    "98:c3_num_class_images: int = 0,\n",
    "99:c3_sample_seed: int = -1,\n",
    "100:c3_save_guidance_scale: float = 7.5,\n",
    "101:c3_save_infer_steps: int = 60,\n",
    "102:c3_save_sample_negative_prompt: str = \"\",\n",
    "103:c3_save_sample_prompt: str = \"\",\n",
    "104:c3_save_sample_template: str = \"\",\n",
    "105:concepts_list=None\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "be957491-0b71-4351-9a3c-35eda7ec9c77",
   "metadata": {},
   "outputs": [],
   "source": [
    "dreambooth_params = [\n",
    "  \"\",\n",
    "  0.9,\n",
    "  0.999,\n",
    "  1e-08,\n",
    "  0.01,\n",
    "  \"default\",\n",
    "  False,\n",
    "  \"\",\n",
    "  \"\",\n",
    "  0.0,\n",
    "  60.0,\n",
    "  1,\n",
    "  True,\n",
    "  False,\n",
    "  \"\",\n",
    "  True,\n",
    "  2e-06,\n",
    "  0.0002,\n",
    "  0.0002,\n",
    "  \"constant\",\n",
    "  500,\n",
    "  75,\n",
    "  0,\n",
    "  \"no\",\n",
    "  \"\",\n",
    "  True,\n",
    "  100,\n",
    "  True,\n",
    "  \"\",\n",
    "  1,\n",
    "  512,\n",
    "  \"\",\n",
    "  1,\n",
    "  True,\n",
    "  500,\n",
    "  500,\n",
    "  True,\n",
    "  False,\n",
    "  False,\n",
    "  \"\",\n",
    "  \"\",\n",
    "  False,\n",
    "  1,\n",
    "  True,\n",
    "  False,\n",
    "  False,\n",
    "  False,\n",
    "  False,\n",
    "  False,\n",
    "  \"\",\n",
    "  \"/opt/ml/input/data/images/\",\n",
    "  7.5,\n",
    "  40,\n",
    "  \"\",\n",
    "  \"a photo of Erwin Rommel\",\n",
    "  \"\",\n",
    "  \"/opt/ml/input/data/images/\",\n",
    "  \"Erwin Rommel\",\n",
    "  \"\",\n",
    "  -1,\n",
    "  1,\n",
    "  0,\n",
    "  -1,\n",
    "  7.5,\n",
    "  40,\n",
    "  \"\",\n",
    "  \"\",\n",
    "  \"\",\n",
    "  \"\",\n",
    "  7.5,\n",
    "  40,\n",
    "  \"\",\n",
    "  \"\",\n",
    "  \"\",\n",
    "  \"\",\n",
    "  \"\",\n",
    "  \"\",\n",
    "  -1,\n",
    "  1,\n",
    "  0,\n",
    "  -1,\n",
    "  7.5,\n",
    "  40,\n",
    "  \"\",\n",
    "  \"\",\n",
    "  \"\",\n",
    "  \"\",\n",
    "  7.5,\n",
    "  40,\n",
    "  \"\",\n",
    "  \"\",\n",
    "  \"\",\n",
    "  \"\",\n",
    "  \"\",\n",
    "  \"\",\n",
    "  -1,\n",
    "  1,\n",
    "  0,\n",
    "  -1,\n",
    "  7.5,\n",
    "  40,\n",
    "  \"\",\n",
    "  \"\",\n",
    "  \"\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b46f3044-4e3d-4e25-a1e5-aaca22926b49",
   "metadata": {},
   "outputs": [],
   "source": [
    "import uuid, json\n",
    "dreambooth_config_id = \"dreambooth_train_config\"\n",
    "dreambooth_config_file =f'{dreambooth_config_id}.json'\n",
    "json.dump(dreambooth_params, open(dreambooth_config_file,'w'), indent=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f09633bd-bdd6-44df-9a0d-7fc1619696d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\"\", 0.9, 0.999, 1e-08, 0.01, \"default\", false, \"\", \"\", 0.0, 60.0, 1, true, false, \"\", true, 2e-06, 0.0002, 0.0002, \"constant\", 500, 75, 0, \"no\", \"\", true, 100, true, \"\", 1, 512, \"\", 1, true, 500, 500, true, false, false, \"\", \"\", false, 1, true, false, false, false, false, false, \"\", \"/opt/ml/input/data/images/\", 7.5, 40, \"\", \"a photo of Erwin Rommel\", \"\", \"/opt/ml/input/data/images/\", \"Erwin Rommel\", \"\", -1, 1, 0, -1, 7.5, 40, \"\", \"\", \"\", \"\", 7.5, 40, \"\", \"\", \"\", \"\", \"\", \"\", -1, 1, 0, -1, 7.5, 40, \"\", \"\", \"\", \"\", 7.5, 40, \"\", \"\", \"\", \"\", \"\", \"\", -1, 1, 0, -1, 7.5, 40, \"\", \"\", \"\"]\n"
     ]
    }
   ],
   "source": [
    "print(json.dumps(dreambooth_params))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "3656f142-057c-4bc9-bd38-ea318b8c4865",
   "metadata": {},
   "outputs": [],
   "source": [
    "#image_uri = '{0}.dkr.ecr.{1}.amazonaws.com/all-in-one-ai-stable-diffusion-webui-training'.format(account_id, region_name)\n",
    "image_uri = '687912291502.dkr.ecr.ap-southeast-1.amazonaws.com/dreambooth-finetuning-v3:latest'\n",
    "models_s3uri = 's3://{0}/stable-diffusion/models/'.format(bucket)\n",
    "dreambooth_s3uri = 's3://{0}/stable-diffusion/dreambooth/'.format(bucket)\n",
    "dreambooth_config_s3uri = 's3://{0}/stable-diffusioni/dreambooth-config/'.format(bucket)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "37afa1cf-56ce-4299-affb-500ae7b17319",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dreambooth_train_config.json\n",
      "s3://sagemaker-ap-southeast-1-687912291502/stable-diffusioni/dreambooth-config/\n",
      "upload: ./dreambooth_train_config.json to s3://sagemaker-ap-southeast-1-687912291502/stable-diffusioni/dreambooth-config/dreambooth_train_config.json\n"
     ]
    }
   ],
   "source": [
    "print(dreambooth_config_file)\n",
    "print(dreambooth_config_s3uri)\n",
    "!aws s3 cp $dreambooth_config_file $dreambooth_config_s3uri"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "38dda204-a307-4776-b907-e8e3548df905",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train-args {\"train_dreambooth_settings\": {\"db_create_new_db_model\": true, \"db_new_model_name\": \"aws-db-new-model\", \"db_new_model_src\": \"768-v-ema.ckpt\", \"db_new_model_scheduler\": \"ddim\", \"db_create_from_hub\": false, \"db_new_model_url\": \"\", \"db_new_model_token\": \"\", \"db_new_model_extract_ema\": false, \"db_model_name\": \"\", \"db_lora_model_name\": \"\", \"db_lora_weight\": 1, \"db_lora_txt_weight\": 1, \"db_train_imagic_only\": false, \"db_use_subdir\": false, \"db_custom_model_name\": \"\", \"db_train_wizard_person\": false, \"db_train_wizard_object\": true, \"db_performance_wizard\": true}}\n",
      "train-task dreambooth\n",
      "sd-models-s3uri s3://sagemaker-ap-southeast-1-687912291502/stable-diffusion/models/\n",
      "db-models-s3uri s3://sagemaker-ap-southeast-1-687912291502/stable-diffusion/dreambooth/\n",
      "ckpt /opt/ml/input/data/models/768-v-ema.ckpt\n",
      "dreambooth-config-id dreambooth_train_config\n",
      "api-endpoint noapi\n"
     ]
    }
   ],
   "source": [
    "def json_encode_hyperparameters(hyperparameters):\n",
    "    for (k, v) in hyperparameters.items():\n",
    "        print(k, v)\n",
    "    \n",
    "    return {k: json.dumps(v) for (k, v) in hyperparameters.items()}\n",
    "\n",
    "train_args = {\n",
    "    'train_dreambooth_settings': {\n",
    "        'db_create_new_db_model': True, \n",
    "        'db_new_model_name': 'aws-db-new-model', \n",
    "        'db_new_model_src': '768-v-ema.ckpt', \n",
    "        'db_new_model_scheduler': 'ddim', \n",
    "        'db_create_from_hub': False, \n",
    "        'db_new_model_url': '', \n",
    "        'db_new_model_token': '', \n",
    "        'db_new_model_extract_ema': False, \n",
    "        'db_model_name': '', \n",
    "        'db_lora_model_name': '', \n",
    "        'db_lora_weight': 1, \n",
    "        'db_lora_txt_weight': 1, \n",
    "        'db_train_imagic_only': False, \n",
    "        'db_use_subdir': False, \n",
    "        'db_custom_model_name': '', \n",
    "        'db_train_wizard_person': False, \n",
    "        'db_train_wizard_object': True, \n",
    "        'db_performance_wizard': True\n",
    "    }\n",
    "}\n",
    "\n",
    "\n",
    "hyperparameters = {\n",
    "    'train-args': json.dumps(train_args),\n",
    "    'train-task': 'dreambooth',\n",
    "    'sd-models-s3uri': models_s3uri,\n",
    "    'db-models-s3uri': dreambooth_s3uri,\n",
    "    'ckpt': '/opt/ml/input/data/models/768-v-ema.ckpt',\n",
    "    'dreambooth-config-id': dreambooth_config_id,\n",
    "    'api-endpoint': 'noapi'\n",
    "}\n",
    "\n",
    "hyperparameters = json_encode_hyperparameters(hyperparameters)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1fb547e0-c9fa-40ec-8acb-68736292803a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s3://sagemaker-ap-southeast-1-687912291502/stable-diffusion/models/\n"
     ]
    }
   ],
   "source": [
    "print(models_s3uri)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9cf52509-8df2-4e35-96ca-0c5738f09887",
   "metadata": {},
   "source": [
    "#################与webui绑定的training BYOC#########################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cccaf4b6-b813-40fa-96d9-10320e37b110",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#!aws s3 cp ./768-v-ema.ckpt   $models_s3uri\n",
    "#!aws s3 cp ./768-v-ema.yaml   $models_s3uri\n",
    "image_uri = '687912291502.dkr.ecr.ap-southeast-1.amazonaws.com/dreambooth-finetuning-v3-with-webui:latest'\n",
    "#image_uri = '687912291502.dkr.ecr.ap-southeast-1.amazonaws.com/dreambooth-finetuning-v3:latest'\n",
    "#image_uri = '687912291502.dkr.ecr.ap-southeast-1.amazonaws.com/dreambooth-finetuning-v3-withw-webui:latest'\n",
    "instance_type = 'ml.g4dn.2xlarge'\n",
    "#image_uri = '687912291502.dkr.ecr.ap-southeast-1.amazonaws.com/dreambooth-finetuning-v3:latest'\n",
    "from sagemaker.estimator import Estimator\n",
    "inputs = {\n",
    "    'images': images_s3uri,\n",
    "    'models': models_s3uri,\n",
    "    'config': dreambooth_config_s3uri\n",
    "}\n",
    "# Please exectute tools/prepare.py with the path to your model files directory.\n",
    "# Make sure the 768-v-ema.ckpt and 768-v-ema.yaml have been uploaded to sd_models_s3uri\n",
    "estimator = Estimator(\n",
    "    role = role,\n",
    "    instance_count=1,\n",
    "    instance_type = instance_type,\n",
    "    image_uri = image_uri,\n",
    "    hyperparameters = hyperparameters\n",
    ")\n",
    "estimator.fit(inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2669338-ea5b-45e9-ab78-a90b8f9b37f9",
   "metadata": {},
   "source": [
    "#################不与webui绑定的training BYOC#########################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "81642194-5e1b-45d6-9e59-1f4c930ecdcc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                           PRE .ipynb_checkpoints/\n",
      "                           PRE dir1/\n",
      "                           PRE test_dir/\n",
      "2023-02-20 09:03:03       3374 helper.py\n",
      "2023-02-20 08:57:04        537 model_index.json\n",
      "2023-02-20 09:03:03       1546 prepare.py\n"
     ]
    }
   ],
   "source": [
    "!aws s3 ls s3://sagemaker-ap-southeast-1-687912291502/dreambooth/trained_models/\n",
    "#!aws s3 rm \"s3://sagemaker-ap-southeast-1-687912291502/dreambooth/trained_models/*\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "32ad6cd8-eece-43d2-b4c8-b210c63b7833",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model_name aws-trained-dreambooth-model\n",
      "mixed_precision fp16\n",
      "pretrained_model_name_or_path runwayml/stable-diffusion-v1-5\n",
      "instance_data_dir /opt/ml/input/data/images/\n",
      "class_data_dir /opt/ml/input/data/images/\n",
      "with_prior_preservation True\n",
      "models_path /opt/ml/output/\n",
      "manul_upload_model_path s3://sagemaker-ap-southeast-1-687912291502/dreambooth/trained_models\n",
      "instance_prompt Erwin\\ Rommel\n",
      "class_prompt a\\ photo\\ of\\ Erwin\\ Rommel\n",
      "resolution 512\n",
      "train_batch_size 1\n",
      "sample_batch_size 1\n",
      "gradient_accumulation_steps 1\n",
      "learning_rate 2e-06\n",
      "lr_scheduler constant\n",
      "lr_warmup_steps 100\n",
      "num_class_images 0\n",
      "max_train_steps 800\n",
      "save_steps 800\n",
      "attention xformers\n",
      "prior_loss_weight 0.5\n",
      "use_ema True\n",
      "train_text_encoder False\n",
      "not_cache_latents True\n",
      "gradient_checkpointing True\n",
      "save_use_epochs False\n",
      "use_8bit_adam False\n",
      "2023-02-27 10:36:15 Starting - Starting the training job...\n",
      "2023-02-27 10:36:38 Starting - Preparing the instances for trainingProfilerReport-1677494174: InProgress\n",
      "......\n",
      "2023-02-27 10:37:41 Downloading - Downloading input data...\n",
      "2023-02-27 10:38:00 Training - Downloading the training image.................................\n",
      "2023-02-27 10:43:40 Training - Training image download completed. Training in progress.......\u001b[34m2023-02-27 10:44:31,167 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2023-02-27 10:44:31,200 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2023-02-27 10:44:31,232 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2023-02-27 10:44:31,243 sagemaker-training-toolkit INFO     Invoking user script\u001b[0m\n",
      "\u001b[34mTraining Env:\u001b[0m\n",
      "\u001b[34m{\n",
      "    \"additional_framework_parameters\": {},\n",
      "    \"channel_input_dirs\": {\n",
      "        \"images\": \"/opt/ml/input/data/images\"\n",
      "    },\n",
      "    \"current_host\": \"algo-1\",\n",
      "    \"current_instance_group\": \"homogeneousCluster\",\n",
      "    \"current_instance_group_hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"current_instance_type\": \"ml.g4dn.4xlarge\",\n",
      "    \"distribution_hosts\": [],\n",
      "    \"distribution_instance_groups\": [],\n",
      "    \"framework_module\": null,\n",
      "    \"hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"hyperparameters\": {\n",
      "        \"attention\": \"xformers\",\n",
      "        \"class_data_dir\": \"/opt/ml/input/data/images/\",\n",
      "        \"class_prompt\": \"a\\\\ photo\\\\ of\\\\ Erwin\\\\ Rommel\",\n",
      "        \"gradient_accumulation_steps\": 1,\n",
      "        \"gradient_checkpointing\": true,\n",
      "        \"instance_data_dir\": \"/opt/ml/input/data/images/\",\n",
      "        \"instance_prompt\": \"Erwin\\\\ Rommel\",\n",
      "        \"learning_rate\": 2e-06,\n",
      "        \"lr_scheduler\": \"constant\",\n",
      "        \"lr_warmup_steps\": 100,\n",
      "        \"manul_upload_model_path\": \"s3://sagemaker-ap-southeast-1-687912291502/dreambooth/trained_models\",\n",
      "        \"max_train_steps\": 800,\n",
      "        \"mixed_precision\": \"fp16\",\n",
      "        \"model_name\": \"aws-trained-dreambooth-model\",\n",
      "        \"models_path\": \"/opt/ml/output/\",\n",
      "        \"not_cache_latents\": true,\n",
      "        \"num_class_images\": 0,\n",
      "        \"pretrained_model_name_or_path\": \"runwayml/stable-diffusion-v1-5\",\n",
      "        \"prior_loss_weight\": 0.5,\n",
      "        \"resolution\": 512,\n",
      "        \"sample_batch_size\": 1,\n",
      "        \"save_steps\": 800,\n",
      "        \"save_use_epochs\": false,\n",
      "        \"train_batch_size\": 1,\n",
      "        \"train_text_encoder\": false,\n",
      "        \"use_8bit_adam\": false,\n",
      "        \"use_ema\": true,\n",
      "        \"with_prior_preservation\": true\n",
      "    },\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {\n",
      "        \"images\": {\n",
      "            \"TrainingInputMode\": \"File\",\n",
      "            \"S3DistributionType\": \"FullyReplicated\",\n",
      "            \"RecordWrapperType\": \"None\"\n",
      "        }\n",
      "    },\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"instance_groups\": [\n",
      "        \"homogeneousCluster\"\n",
      "    ],\n",
      "    \"instance_groups_dict\": {\n",
      "        \"homogeneousCluster\": {\n",
      "            \"instance_group_name\": \"homogeneousCluster\",\n",
      "            \"instance_type\": \"ml.g4dn.4xlarge\",\n",
      "            \"hosts\": [\n",
      "                \"algo-1\"\n",
      "            ]\n",
      "        }\n",
      "    },\n",
      "    \"is_hetero\": false,\n",
      "    \"is_master\": true,\n",
      "    \"is_modelparallel_enabled\": null,\n",
      "    \"is_smddpmprun_installed\": false,\n",
      "    \"job_name\": \"dreambooth-finetuning-v3-2023-02-27-10-36-14-292\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"/opt/ml/code\",\n",
      "    \"module_name\": \"train\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 16,\n",
      "    \"num_gpus\": 1,\n",
      "    \"num_neurons\": 0,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-1\",\n",
      "        \"current_instance_type\": \"ml.g4dn.4xlarge\",\n",
      "        \"current_group_name\": \"homogeneousCluster\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\"\n",
      "        ],\n",
      "        \"instance_groups\": [\n",
      "            {\n",
      "                \"instance_group_name\": \"homogeneousCluster\",\n",
      "                \"instance_type\": \"ml.g4dn.4xlarge\",\n",
      "                \"hosts\": [\n",
      "                    \"algo-1\"\n",
      "                ]\n",
      "            }\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"train.py\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34mEnvironment variables:\u001b[0m\n",
      "\u001b[34mSM_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[34mSM_HPS={\"attention\":\"xformers\",\"class_data_dir\":\"/opt/ml/input/data/images/\",\"class_prompt\":\"a\\\\ photo\\\\ of\\\\ Erwin\\\\ Rommel\",\"gradient_accumulation_steps\":1,\"gradient_checkpointing\":true,\"instance_data_dir\":\"/opt/ml/input/data/images/\",\"instance_prompt\":\"Erwin\\\\ Rommel\",\"learning_rate\":2e-06,\"lr_scheduler\":\"constant\",\"lr_warmup_steps\":100,\"manul_upload_model_path\":\"s3://sagemaker-ap-southeast-1-687912291502/dreambooth/trained_models\",\"max_train_steps\":800,\"mixed_precision\":\"fp16\",\"model_name\":\"aws-trained-dreambooth-model\",\"models_path\":\"/opt/ml/output/\",\"not_cache_latents\":true,\"num_class_images\":0,\"pretrained_model_name_or_path\":\"runwayml/stable-diffusion-v1-5\",\"prior_loss_weight\":0.5,\"resolution\":512,\"sample_batch_size\":1,\"save_steps\":800,\"save_use_epochs\":false,\"train_batch_size\":1,\"train_text_encoder\":false,\"use_8bit_adam\":false,\"use_ema\":true,\"with_prior_preservation\":true}\u001b[0m\n",
      "\u001b[34mSM_USER_ENTRY_POINT=train.py\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_PARAMS={}\u001b[0m\n",
      "\u001b[34mSM_RESOURCE_CONFIG={\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.g4dn.4xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g4dn.4xlarge\"}],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[34mSM_INPUT_DATA_CONFIG={\"images\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}}\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[34mSM_CHANNELS=[\"images\"]\u001b[0m\n",
      "\u001b[34mSM_CURRENT_HOST=algo-1\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_TYPE=ml.g4dn.4xlarge\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_GROUP=homogeneousCluster\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_GROUP_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_INSTANCE_GROUPS=[\"homogeneousCluster\"]\u001b[0m\n",
      "\u001b[34mSM_INSTANCE_GROUPS_DICT={\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g4dn.4xlarge\"}}\u001b[0m\n",
      "\u001b[34mSM_DISTRIBUTION_INSTANCE_GROUPS=[]\u001b[0m\n",
      "\u001b[34mSM_IS_HETERO=false\u001b[0m\n",
      "\u001b[34mSM_MODULE_NAME=train\u001b[0m\n",
      "\u001b[34mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_MODULE=\u001b[0m\n",
      "\u001b[34mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[34mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[34mSM_NUM_CPUS=16\u001b[0m\n",
      "\u001b[34mSM_NUM_GPUS=1\u001b[0m\n",
      "\u001b[34mSM_NUM_NEURONS=0\u001b[0m\n",
      "\u001b[34mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[34mSM_MODULE_DIR=/opt/ml/code\u001b[0m\n",
      "\u001b[34mSM_TRAINING_ENV={\"additional_framework_parameters\":{},\"channel_input_dirs\":{\"images\":\"/opt/ml/input/data/images\"},\"current_host\":\"algo-1\",\"current_instance_group\":\"homogeneousCluster\",\"current_instance_group_hosts\":[\"algo-1\"],\"current_instance_type\":\"ml.g4dn.4xlarge\",\"distribution_hosts\":[],\"distribution_instance_groups\":[],\"framework_module\":null,\"hosts\":[\"algo-1\"],\"hyperparameters\":{\"attention\":\"xformers\",\"class_data_dir\":\"/opt/ml/input/data/images/\",\"class_prompt\":\"a\\\\ photo\\\\ of\\\\ Erwin\\\\ Rommel\",\"gradient_accumulation_steps\":1,\"gradient_checkpointing\":true,\"instance_data_dir\":\"/opt/ml/input/data/images/\",\"instance_prompt\":\"Erwin\\\\ Rommel\",\"learning_rate\":2e-06,\"lr_scheduler\":\"constant\",\"lr_warmup_steps\":100,\"manul_upload_model_path\":\"s3://sagemaker-ap-southeast-1-687912291502/dreambooth/trained_models\",\"max_train_steps\":800,\"mixed_precision\":\"fp16\",\"model_name\":\"aws-trained-dreambooth-model\",\"models_path\":\"/opt/ml/output/\",\"not_cache_latents\":true,\"num_class_images\":0,\"pretrained_model_name_or_path\":\"runwayml/stable-diffusion-v1-5\",\"prior_loss_weight\":0.5,\"resolution\":512,\"sample_batch_size\":1,\"save_steps\":800,\"save_use_epochs\":false,\"train_batch_size\":1,\"train_text_encoder\":false,\"use_8bit_adam\":false,\"use_ema\":true,\"with_prior_preservation\":true},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{\"images\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}},\"input_dir\":\"/opt/ml/input\",\"instance_groups\":[\"homogeneousCluster\"],\"instance_groups_dict\":{\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g4dn.4xlarge\"}},\"is_hetero\":false,\"is_master\":true,\"is_modelparallel_enabled\":null,\"is_smddpmprun_installed\":false,\"job_name\":\"dreambooth-finetuning-v3-2023-02-27-10-36-14-292\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"/opt/ml/code\",\"module_name\":\"train\",\"network_interface_name\":\"eth0\",\"num_cpus\":16,\"num_gpus\":1,\"num_neurons\":0,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.g4dn.4xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g4dn.4xlarge\"}],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"train.py\"}\u001b[0m\n",
      "\u001b[34mSM_USER_ARGS=[\"--attention\",\"xformers\",\"--class_data_dir\",\"/opt/ml/input/data/images/\",\"--class_prompt\",\"a\\\\ photo\\\\ of\\\\ Erwin\\\\ Rommel\",\"--gradient_accumulation_steps\",\"1\",\"--gradient_checkpointing\",\"True\",\"--instance_data_dir\",\"/opt/ml/input/data/images/\",\"--instance_prompt\",\"Erwin\\\\ Rommel\",\"--learning_rate\",\"2e-06\",\"--lr_scheduler\",\"constant\",\"--lr_warmup_steps\",\"100\",\"--manul_upload_model_path\",\"s3://sagemaker-ap-southeast-1-687912291502/dreambooth/trained_models\",\"--max_train_steps\",\"800\",\"--mixed_precision\",\"fp16\",\"--model_name\",\"aws-trained-dreambooth-model\",\"--models_path\",\"/opt/ml/output/\",\"--not_cache_latents\",\"True\",\"--num_class_images\",\"0\",\"--pretrained_model_name_or_path\",\"runwayml/stable-diffusion-v1-5\",\"--prior_loss_weight\",\"0.5\",\"--resolution\",\"512\",\"--sample_batch_size\",\"1\",\"--save_steps\",\"800\",\"--save_use_epochs\",\"False\",\"--train_batch_size\",\"1\",\"--train_text_encoder\",\"False\",\"--use_8bit_adam\",\"False\",\"--use_ema\",\"True\",\"--with_prior_preservation\",\"True\"]\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[34mSM_CHANNEL_IMAGES=/opt/ml/input/data/images\u001b[0m\n",
      "\u001b[34mSM_HP_ATTENTION=xformers\u001b[0m\n",
      "\u001b[34mSM_HP_CLASS_DATA_DIR=/opt/ml/input/data/images/\u001b[0m\n",
      "\u001b[34mSM_HP_CLASS_PROMPT=a\\ photo\\ of\\ Erwin\\ Rommel\u001b[0m\n",
      "\u001b[34mSM_HP_GRADIENT_ACCUMULATION_STEPS=1\u001b[0m\n",
      "\u001b[34mSM_HP_GRADIENT_CHECKPOINTING=true\u001b[0m\n",
      "\u001b[34mSM_HP_INSTANCE_DATA_DIR=/opt/ml/input/data/images/\u001b[0m\n",
      "\u001b[34mSM_HP_INSTANCE_PROMPT=Erwin\\ Rommel\u001b[0m\n",
      "\u001b[34mSM_HP_LEARNING_RATE=2e-06\u001b[0m\n",
      "\u001b[34mSM_HP_LR_SCHEDULER=constant\u001b[0m\n",
      "\u001b[34mSM_HP_LR_WARMUP_STEPS=100\u001b[0m\n",
      "\u001b[34mSM_HP_MANUL_UPLOAD_MODEL_PATH=s3://sagemaker-ap-southeast-1-687912291502/dreambooth/trained_models\u001b[0m\n",
      "\u001b[34mSM_HP_MAX_TRAIN_STEPS=800\u001b[0m\n",
      "\u001b[34mSM_HP_MIXED_PRECISION=fp16\u001b[0m\n",
      "\u001b[34mSM_HP_MODEL_NAME=aws-trained-dreambooth-model\u001b[0m\n",
      "\u001b[34mSM_HP_MODELS_PATH=/opt/ml/output/\u001b[0m\n",
      "\u001b[34mSM_HP_NOT_CACHE_LATENTS=true\u001b[0m\n",
      "\u001b[34mSM_HP_NUM_CLASS_IMAGES=0\u001b[0m\n",
      "\u001b[34mSM_HP_PRETRAINED_MODEL_NAME_OR_PATH=runwayml/stable-diffusion-v1-5\u001b[0m\n",
      "\u001b[34mSM_HP_PRIOR_LOSS_WEIGHT=0.5\u001b[0m\n",
      "\u001b[34mSM_HP_RESOLUTION=512\u001b[0m\n",
      "\u001b[34mSM_HP_SAMPLE_BATCH_SIZE=1\u001b[0m\n",
      "\u001b[34mSM_HP_SAVE_STEPS=800\u001b[0m\n",
      "\u001b[34mSM_HP_SAVE_USE_EPOCHS=false\u001b[0m\n",
      "\u001b[34mSM_HP_TRAIN_BATCH_SIZE=1\u001b[0m\n",
      "\u001b[34mSM_HP_TRAIN_TEXT_ENCODER=false\u001b[0m\n",
      "\u001b[34mSM_HP_USE_8BIT_ADAM=false\u001b[0m\n",
      "\u001b[34mSM_HP_USE_EMA=true\u001b[0m\n",
      "\u001b[34mSM_HP_WITH_PRIOR_PRESERVATION=true\u001b[0m\n",
      "\u001b[34mPYTHONPATH=/opt/ml/code:/opt/conda/bin:/opt/conda/lib/python39.zip:/opt/conda/lib/python3.9:/opt/conda/lib/python3.9/lib-dynload:/opt/conda/lib/python3.9/site-packages:/opt/ml/code/repositories/xformers\u001b[0m\n",
      "\u001b[34mInvoking script with the following command:\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/python train.py --attention xformers --class_data_dir /opt/ml/input/data/images/ --class_prompt a\\ photo\\ of\\ Erwin\\ Rommel --gradient_accumulation_steps 1 --gradient_checkpointing True --instance_data_dir /opt/ml/input/data/images/ --instance_prompt Erwin\\ Rommel --learning_rate 2e-06 --lr_scheduler constant --lr_warmup_steps 100 --manul_upload_model_path s3://sagemaker-ap-southeast-1-687912291502/dreambooth/trained_models --max_train_steps 800 --mixed_precision fp16 --model_name aws-trained-dreambooth-model --models_path /opt/ml/output/ --not_cache_latents True --num_class_images 0 --pretrained_model_name_or_path runwayml/stable-diffusion-v1-5 --prior_loss_weight 0.5 --resolution 512 --sample_batch_size 1 --save_steps 800 --save_use_epochs False --train_batch_size 1 --train_text_encoder False --use_8bit_adam False --use_ema True --with_prior_preservation True\u001b[0m\n",
      "\u001b[34m2023-02-27 10:44:31,244 sagemaker-training-toolkit INFO     Exceptions not imported for SageMaker Debugger as it is not installed.\u001b[0m\n",
      "\u001b[34m2023-02-27 10:44:31,244 sagemaker-training-toolkit INFO     Exceptions not imported for SageMaker TF as Tensorflow is not installed.\u001b[0m\n",
      "\u001b[34mA matching Triton is not available, some optimizations will not be enabled.\u001b[0m\n",
      "\u001b[34mError caught was: module 'triton.language' has no attribute 'constexpr'\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/transformers/generation_utils.py:24: FutureWarning: Importing `GenerationMixin` from `src/transformers/generation_utils.py` is deprecated and will be removed in Transformers v5. Import as `from transformers import GenerationMixin` instead.\n",
      "  warnings.warn(\u001b[0m\n",
      "\u001b[34mPatching transformers to fix kwargs errors.\u001b[0m\n",
      "\u001b[34m{'manul_upload_model_path': 's3://sagemaker-ap-southeast-1-687912291502/dreambooth/trained_models', 'model_name': 'aws-trained-dreambooth-model', 'models_path': '/opt/ml/output/', 'use_lora': False, 'use_cpu': False, 'lora_models_path': None, 'pretrained_model_name_or_path': 'runwayml/stable-diffusion-v1-5', 'pretrained_vae_name_or_path': None, 'revision': None, 'tokenizer_name': None, 'instance_data_dir': '/opt/ml/input/data/images/', 'class_data_dir': '/opt/ml/input/data/images/', 'instance_prompt': 'Erwin\\\\ Rommel', 'class_prompt': 'a\\\\ photo\\\\ of\\\\ Erwin\\\\ Rommel', 'pad_tokens': False, 'with_prior_preservation': True, 'save_use_global_counts': False, 'save_use_epochs': True, 'prior_loss_weight': 0.5, 'num_class_images': 0, 'output_dir': 'text-inversion-model', 'seed': -1, 'resolution': 512, 'center_crop': False, 'train_text_encoder': 'False', 'train_batch_size': 1, 'sample_batch_size': 1, 'num_train_epochs': 1, 'max_train_steps': 800, 'epoch': 0, 'save_steps': 800, 'gradient_accumulation_steps': 1, 'gradient_checkpointing': True, 'learning_rate': 2e-06, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 100, 'use_8bit_adam': True, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'push_to_hub': False, 'hub_token': None, 'hub_model_id': None, 'logging_dir': 'logs', 'mixed_precision': 'fp16', 'not_cache_latents': 'True', 'hflip': False, 'local_rank': -1, 'concepts_list': [{'instance_prompt': 'Erwin\\\\ Rommel', 'class_prompt': 'a\\\\ photo\\\\ of\\\\ Erwin\\\\ Rommel', 'instance_data_dir': '/opt/ml/input/data/images/', 'class_data_dir': '/opt/ml/input/data/images/', 'num_class_images': 0, 'instance_token': '', 'class_token': '', 'class_negative_prompt': '', 'class_guidance_scale': 7.5, 'class_infer_steps': 60}], 'use_ema': True, 'max_token_length': 75, 'half_model': False, 'attention': 'xformers', 'shuffle_tags': False}\u001b[0m\n",
      "\u001b[34mReplace CrossAttention.forward to use xformers\u001b[0m\n",
      "\u001b[34mChecking concept: {'instance_prompt': 'Erwin\\\\ Rommel', 'class_prompt': 'a\\\\ photo\\\\ of\\\\ Erwin\\\\ Rommel', 'instance_data_dir': '/opt/ml/input/data/images/', 'class_data_dir': '/opt/ml/input/data/images/', 'num_class_images': 0, 'instance_token': '', 'class_token': '', 'class_negative_prompt': '', 'class_guidance_scale': 7.5, 'class_infer_steps': 60}\u001b[0m\n",
      "\u001b[34mConcept requires 0 images.\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\n",
      "  File \"/opt/conda/lib/python3.9/site-packages/transformers/utils/hub.py\", line 409, in cached_file\n",
      "    resolved_file = hf_hub_download(\n",
      "  File \"/opt/conda/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py\", line 124, in _inner_fn\n",
      "    return fn(*args, **kwargs)\n",
      "  File \"/opt/conda/lib/python3.9/site-packages/huggingface_hub/file_download.py\", line 1212, in hf_hub_download\n",
      "    raise LocalEntryNotFoundError(\u001b[0m\n",
      "\u001b[34mhuggingface_hub.utils._errors.LocalEntryNotFoundError: Connection error, and we cannot find the requested files in the disk cache. Please try again or make sure your Internet connection is on.\u001b[0m\n",
      "\u001b[34mDuring handling of the above exception, another exception occurred:\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\n",
      "  File \"/opt/ml/code/train.py\", line 1274, in <module>\n",
      "    main(args=args, memory_record={}, use_subdir=False, lora_model=None, lora_alpha=1.0, lora_txt_alpha=1.0, custom_model_name=\"\")\n",
      "  File \"/opt/ml/code/train.py\", line 630, in main\n",
      "    tokenizer = AutoTokenizer.from_pretrained(\n",
      "  File \"/opt/conda/lib/python3.9/site-packages/transformers/models/auto/tokenization_auto.py\", line 597, in from_pretrained\n",
      "    config = AutoConfig.from_pretrained(\n",
      "  File \"/opt/conda/lib/python3.9/site-packages/transformers/models/auto/configuration_auto.py\", line 809, in from_pretrained\n",
      "    config_dict, unused_kwargs = PretrainedConfig.get_config_dict(pretrained_model_name_or_path, **kwargs)\n",
      "  File \"/opt/conda/lib/python3.9/site-packages/transformers/configuration_utils.py\", line 559, in get_config_dict\n",
      "    config_dict, kwargs = cls._get_config_dict(pretrained_model_name_or_path, **kwargs)\n",
      "  File \"/opt/conda/lib/python3.9/site-packages/transformers/configuration_utils.py\", line 614, in _get_config_dict\n",
      "    resolved_config_file = cached_file(\n",
      "  File \"/opt/conda/lib/python3.9/site-packages/transformers/utils/hub.py\", line 443, in cached_file\n",
      "    raise EnvironmentError(\u001b[0m\n",
      "\u001b[34mOSError: We couldn't connect to 'https://huggingface.co' to load this file, couldn't find it in the cached files and it looks like runwayml/stable-diffusion-v1-5 is not the path to a directory containing a file named tokenizer/config.json.\u001b[0m\n",
      "\u001b[34mCheckout your internet connection or see how to run the library in offline mode at 'https://huggingface.co/docs/transformers/installation#offline-mode'.\u001b[0m\n",
      "\u001b[34m2023-02-27 10:44:57,406 sagemaker-training-toolkit ERROR    Reporting training FAILURE\u001b[0m\n",
      "\u001b[34m2023-02-27 10:44:57,406 sagemaker-training-toolkit ERROR    ExecuteUserScriptError:\u001b[0m\n",
      "\u001b[34mExitCode 1\u001b[0m\n",
      "\u001b[34mErrorMessage \"\"\u001b[0m\n",
      "\u001b[34mCommand \"/opt/conda/bin/python train.py --attention xformers --class_data_dir /opt/ml/input/data/images/ --class_prompt a\\ photo\\ of\\ Erwin\\ Rommel --gradient_accumulation_steps 1 --gradient_checkpointing True --instance_data_dir /opt/ml/input/data/images/ --instance_prompt Erwin\\ Rommel --learning_rate 2e-06 --lr_scheduler constant --lr_warmup_steps 100 --manul_upload_model_path s3://sagemaker-ap-southeast-1-687912291502/dreambooth/trained_models --max_train_steps 800 --mixed_precision fp16 --model_name aws-trained-dreambooth-model --models_path /opt/ml/output/ --not_cache_latents True --num_class_images 0 --pretrained_model_name_or_path runwayml/stable-diffusion-v1-5 --prior_loss_weight 0.5 --resolution 512 --sample_batch_size 1 --save_steps 800 --save_use_epochs False --train_batch_size 1 --train_text_encoder False --use_8bit_adam False --use_ema True --with_prior_preservation True\"\u001b[0m\n",
      "\u001b[34m2023-02-27 10:44:57,406 sagemaker-training-toolkit ERROR    Encountered exit_code 1\u001b[0m\n",
      "\n",
      "2023-02-27 10:45:20 Uploading - Uploading generated training model\n",
      "2023-02-27 10:45:20 Failed - Training job failed\n"
     ]
    },
    {
     "ename": "UnexpectedStatusException",
     "evalue": "Error for Training job dreambooth-finetuning-v3-2023-02-27-10-36-14-292: Failed. Reason: AlgorithmError: ExecuteUserScriptError:\nExitCode 1\nErrorMessage \"\"\nCommand \"/opt/conda/bin/python train.py --attention xformers --class_data_dir /opt/ml/input/data/images/ --class_prompt a\\ photo\\ of\\ Erwin\\ Rommel --gradient_accumulation_steps 1 --gradient_checkpointing True --instance_data_dir /opt/ml/input/data/images/ --instance_prompt Erwin\\ Rommel --learning_rate 2e-06 --lr_scheduler constant --lr_warmup_steps 100 --manul_upload_model_path s3://sagemaker-ap-southeast-1-687912291502/dreambooth/trained_models --max_train_steps 800 --mixed_precision fp16 --model_name aws-trained-dreambooth-model --models_path /opt/ml/output/ --not_cache_latents True --num_class_images 0 --pretrained_model_name_or_path runwayml/stable-diffusion-v1-5 --prior_loss_weight 0.5 --resolution 512 --sample_batch_size 1 --save_steps 800 --save_use_epochs False --train_batch_size 1 --train_text_encoder False --use_8bit_adam False --use_ema True --with_prior_preservation True\", exit code: 1",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mUnexpectedStatusException\u001b[0m                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-75-238058dfea96>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     76\u001b[0m     \u001b[0menvironment\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menvironment\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m )\n\u001b[0;32m---> 78\u001b[0;31m \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/sagemaker/workflow/pipeline_context.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    270\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0m_StepArguments\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mretrieve_caller_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself_instance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_func\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    271\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 272\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mrun_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    273\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    274\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/sagemaker/estimator.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, inputs, wait, logs, job_name, experiment_config)\u001b[0m\n\u001b[1;32m   1126\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjobs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlatest_training_job\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1127\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1128\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlatest_training_job\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1130\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_compilation_job_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/sagemaker/estimator.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, logs)\u001b[0m\n\u001b[1;32m   2240\u001b[0m         \u001b[0;31m# If logs are requested, call logs_for_jobs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2241\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlogs\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m\"None\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2242\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msagemaker_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogs_for_job\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjob_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlog_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2243\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2244\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msagemaker_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait_for_job\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjob_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/sagemaker/session.py\u001b[0m in \u001b[0;36mlogs_for_job\u001b[0;34m(self, job_name, wait, poll, log_type)\u001b[0m\n\u001b[1;32m   4070\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4071\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4072\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_job_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdescription\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"TrainingJobStatus\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   4073\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mdot\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4074\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/sagemaker/session.py\u001b[0m in \u001b[0;36m_check_job_status\u001b[0;34m(self, job, desc, status_key_name)\u001b[0m\n\u001b[1;32m   3604\u001b[0m                 \u001b[0mmessage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3605\u001b[0m                 \u001b[0mallowed_statuses\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"Completed\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Stopped\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3606\u001b[0;31m                 \u001b[0mactual_status\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3607\u001b[0m             )\n\u001b[1;32m   3608\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mUnexpectedStatusException\u001b[0m: Error for Training job dreambooth-finetuning-v3-2023-02-27-10-36-14-292: Failed. Reason: AlgorithmError: ExecuteUserScriptError:\nExitCode 1\nErrorMessage \"\"\nCommand \"/opt/conda/bin/python train.py --attention xformers --class_data_dir /opt/ml/input/data/images/ --class_prompt a\\ photo\\ of\\ Erwin\\ Rommel --gradient_accumulation_steps 1 --gradient_checkpointing True --instance_data_dir /opt/ml/input/data/images/ --instance_prompt Erwin\\ Rommel --learning_rate 2e-06 --lr_scheduler constant --lr_warmup_steps 100 --manul_upload_model_path s3://sagemaker-ap-southeast-1-687912291502/dreambooth/trained_models --max_train_steps 800 --mixed_precision fp16 --model_name aws-trained-dreambooth-model --models_path /opt/ml/output/ --not_cache_latents True --num_class_images 0 --pretrained_model_name_or_path runwayml/stable-diffusion-v1-5 --prior_loss_weight 0.5 --resolution 512 --sample_batch_size 1 --save_steps 800 --save_use_epochs False --train_batch_size 1 --train_text_encoder False --use_8bit_adam False --use_ema True --with_prior_preservation True\", exit code: 1"
     ]
    }
   ],
   "source": [
    "import json\n",
    "def json_encode_hyperparameters(hyperparameters):\n",
    "    for (k, v) in hyperparameters.items():\n",
    "        print(k, v)\n",
    "    \n",
    "    return {k: json.dumps(v) for (k, v) in hyperparameters.items()}\n",
    "\n",
    "\n",
    "\n",
    "image_uri = '687912291502.dkr.ecr.ap-southeast-1.amazonaws.com/dreambooth-finetuning-v3:latest'\n",
    "instance_type = 'ml.g4dn.4xlarge'\n",
    "\n",
    "instance_prompt=\"Erwin\\ Rommel\"\n",
    "prior_preservation_class_prompt=\"a\\ photo\\ of\\ Erwin\\ Rommel\"\n",
    "s3_model_output_location='s3://{}/{}/{}'.format(bucket, 'dreambooth', 'trained_models')\n",
    "#model_name=\"CompVis/stable-diffusion-v1-4\"\n",
    "model_name=\"runwayml/stable-diffusion-v1-5\"\n",
    "#model_name=\"ClueAI/ChatYuan-large-v1\"\n",
    "#model_name=\"stabilityai/stable-diffusion-2\"\n",
    "instance_dir=\"/opt/ml/input/data/images/\"\n",
    "class_dir=\"/opt/ml/input/data/images/\"\n",
    "\n",
    "\n",
    "\n",
    "environment = {\n",
    "    'PYTORCH_CUDA_ALLOC_CONF':'max_split_size_mb:32',\n",
    "    'LD_LIBRARY_PATH':\"${LD_LIBRARY_PATH}:/opt/conda/lib/\"\n",
    "}\n",
    "\n",
    "hyperparameters = {\n",
    "                    'model_name':'aws-trained-dreambooth-model',\n",
    "                    'mixed_precision':'fp16',\n",
    "                    'pretrained_model_name_or_path': model_name, \n",
    "                    'instance_data_dir':instance_dir,\n",
    "                    'class_data_dir':class_dir,\n",
    "                    'with_prior_preservation':True,\n",
    "                    #'models_path': '/opt/ml/model/',\n",
    "                    'models_path': '/opt/ml/output/',\n",
    "                    'manul_upload_model_path':s3_model_output_location,\n",
    "                    'instance_prompt': instance_prompt, \n",
    "                    'class_prompt':prior_preservation_class_prompt,\n",
    "                    'resolution':512,\n",
    "                    'train_batch_size':1,\n",
    "                    'sample_batch_size': 1,\n",
    "                    'gradient_accumulation_steps':1,\n",
    "                    'learning_rate':2e-06,\n",
    "                    'lr_scheduler':'constant',\n",
    "                    'lr_warmup_steps':100,\n",
    "                    'num_class_images':0,\n",
    "                    'max_train_steps':800,\n",
    "                    'save_steps':800,\n",
    "                    #'attention':'flash_attention',\n",
    "                    'attention':'xformers',\n",
    "                    'prior_loss_weight': 0.5,\n",
    "                    'use_ema':True,\n",
    "                    'train_text_encoder':False,\n",
    "                    'not_cache_latents':True,\n",
    "                    'gradient_checkpointing':True,\n",
    "                    'save_use_epochs': False,\n",
    "                    'use_8bit_adam': False\n",
    "}\n",
    "\n",
    "hyperparameters = json_encode_hyperparameters(hyperparameters)\n",
    "\n",
    "from sagemaker.estimator import Estimator\n",
    "inputs = {\n",
    "    'images': images_s3uri\n",
    "}\n",
    "\n",
    "estimator = Estimator(\n",
    "    role = role,\n",
    "    instance_count=1,\n",
    "    instance_type = instance_type,\n",
    "    image_uri = image_uri,\n",
    "    hyperparameters = hyperparameters,\n",
    "    environment = environment\n",
    ")\n",
    "estimator.fit(inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "bd181b4e-f435-4dca-842a-444d083fdf3c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model artifact saved at:\n",
      " s3://sagemaker-ap-southeast-1-687912291502/dreambooth-finetuning-v3-2023-02-20-03-54-22-434/output/model.tar.gz\n"
     ]
    }
   ],
   "source": [
    "dreambooth_model_data = estimator.model_data\n",
    "print(\"Model artifact saved at:\\n\", dreambooth_model_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "21b90beb-3b46-479d-a933-9540b0723331",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-02-20 09:56:38        601 config.json\n",
      "2023-02-20 09:56:37  167402961 diffusion_pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "!aws s3 ls s3://sagemaker-ap-southeast-1-687912291502/dreambooth/trained_models/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7279d922-766c-4a71-8ed4-363467c9c817",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "instance_type": "ml.m5.large",
  "kernelspec": {
   "display_name": "Python 3 (Data Science)",
   "language": "python",
   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:ap-southeast-1:492261229750:image/datascience-1.0"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}