Dalle2 zoom outpainting.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4f82f617",
   "metadata": {},
   "source": [
    "# Generate the images in dalle2\n",
    " 1. Generate an image in dalle2. Inspect the html source and download the webp file without the OpenAI watermark in the corner. Name the file `01.webp`.\n",
    " 2. Run the below line to zoom out on the image.\n",
    " 3. Upload `zoomed.png` to dalle2. \"erase\" a spot on the outside of the image. The prompt can be modified to change the content of the outfilled portion of the image.\n",
    " 4. Repeat steps 1-4 until you have the desired results. Name the files 02.webp, 03.webp, etc...\n",
    " 5. Continue to the next section below \"Animating the results\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8025bf0",
   "metadata": {},
   "outputs": [],
   "source": [
    "!convert 01.webp -gravity center -background none -extent 2560x2560 zoomed.png"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "1f3d815b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-07-18T03:16:08.026534Z",
     "start_time": "2022-07-18T03:16:07.580245Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "from PIL import Image\n",
    "# import ffmpeg\n",
    "from subprocess import check_output, STDOUT\n",
    "\n",
    "# TODO pull these functions out into helpers.py\n",
    "\n",
    "def get_percentage_decrease(width1, width2, percent):\n",
    "    return width2*math.exp(math.log(width1/width2) * (1-percent))\n",
    "\n",
    "def crop(in_fn, out_fn, new_size):\n",
    "    im = Image.open(in_fn)\n",
    "    remove = int(math.ceil((im.width-new_size)/2)) * 2\n",
    "    size = im.width - remove\n",
    "    im2 = im.crop(((im.width-size)//2, (im.height-size)//2, (im.width+size)//2, (im.height+size)//2))\n",
    "    im2.save(out_fn)\n",
    "        \n",
    "def resize(in_fn, out_fn, new_size, rs_type=None):\n",
    "    \"\"\"rs_type: Image.Resampling.NEAREST is no antialiasing fastest but noisy results,\n",
    "                Image.Resampling.BOX is good for a little smoothing and only slightly slower,\n",
    "                Image.Dither.FLOYDSTEINBERG is even more smoothing and a little slower still\n",
    "    \"\"\"\n",
    "    if not rs_type:\n",
    "        rs_type = Image.Resampling.BOX\n",
    "    im = Image.open(in_fn)\n",
    "    im = im.resize((new_size,new_size), rs_type)\n",
    "    im.save(out_fn)\n",
    "    \n",
    "def extend(in_fn, out_fn, new_size):\n",
    "    im = Image.open(in_fn)\n",
    "    im = im.convert('RGBA')\n",
    "    width, height = im.size\n",
    "    x1 = (new_size - width) // 2\n",
    "    y1 = (new_size - height) // 2\n",
    "    new_image = Image.new('RGBA', (new_size, new_size), (0, 0, 0, 0))\n",
    "    new_image.paste(im, (x1, y1, x1 + width, y1 + height))\n",
    "    new_image.save(out_fn)\n",
    "    \n",
    "# THIS IS BROKEN, USE IM for now\n",
    "# def overlay(bottom, top, out_fn):\n",
    "#     im = Image.open(bottom).convert('RGBA')\n",
    "#     im2 = Image.open(top).convert('RGBA')\n",
    "#     im.paste(im2, (0, 0), im2)\n",
    "#     im.save(out_fn)\n",
    "    \n",
    "def overlay(bottom, top, out_fn):\n",
    "    check_output(f'convert {bottom} {top} -gravity center -background None -layers Flatten {out_fn}', shell=True, stderr=STDOUT)\n",
    "\n",
    "def animate_between(i, delay=10, n=2560, final_size=400, mode='gif', fps=60, filename='final', frames=48):\n",
    "    # commented this out for now because it makes the transition glitches worse not better\n",
    "    # couldn't reproduce this in PIL...\n",
    "#     check_output(f'convert {i-1:02d}.webp -alpha set -virtual-pixel transparent -channel A -blur 0x8 -level 50%,100% +channel soft_edge.png', shell=True, stderr=STDOUT)\n",
    "    resize(f'{i:02d}.webp', 'resized.png', n)\n",
    "    # commented these out for now because they make the transition glitches worse not better\n",
    "#     extend('soft_edge.png', 'next.png', n)\n",
    "#     overlay('resized.png', 'next.png', 'resized2.png')\n",
    "#     resize('resized2.png', f'cropped_{i}{frames:03d}.png', final_size)\n",
    "    resize('resized.png', f'cropped_{i}{frames:03d}.png', final_size)\n",
    "    name = 'resized.png'\n",
    "    for j in reversed(range(1, frames)): # start with 0 or 1?\n",
    "        new_size = int(get_percentage_decrease(n, 1024, (frames-j) / frames))\n",
    "        print(f'{j}={new_size}', end=', ')\n",
    "        crop(name, 'cropped.png', new_size)\n",
    "        resize('cropped.png', f'cropped_{i}{j:03d}.png', final_size)\n",
    "        if name == 'resized.png':\n",
    "            name = 'cropped.png'\n",
    "    if mode == 'gif':\n",
    "        check_output(f'convert cropped_*.png -set delay {delay} movie_{i:03d}.gif', shell=True, stderr=STDOUT)\n",
    "    elif mode == 'mp4':\n",
    "        # TODO: replace with python ffmpeg code\n",
    "        check_output(f'ffmpeg -r {fps} -i cropped_{i}%03d.png -y -an movie_{i:03d}.mp4', shell=True, stderr=STDOUT)\n",
    "    check_output('rm cropped_*.png; rm resized.png; rm soft_edge.png; rm resized2.png; rm next.png; rm cropped.png', shell=True, stderr=STDOUT)\n",
    "    print('')\n",
    "    \n",
    "def generate_full_animation(num_dalle, min_dalle=1, delay=10, n=2560, final_size=400, mode='mp4', fps=60, filename='final', frames=48):\n",
    "    \"\"\"Generate a full animation.\n",
    "    \n",
    "        num_dalle: the number of dalle outpaint files you created\n",
    "        min_dalle: the last image that should be processed. If you want to start off at 1.webp, this should be set to 1\n",
    "        delay: used in gif mode to specify the delay between frames\n",
    "        n: the outpaint size specified in step 1 above\n",
    "        final_size: final dimensions of animation\n",
    "        mode: can be either 'gif' or 'mp4'\n",
    "        fps: used in mp4 mode to set the speed of the video\n",
    "        filename: The prefix used for the final output filename\n",
    "        frames: used in mp4 mode to specify the number of frames generated at each stage of the process\n",
    "    \"\"\"\n",
    "    # TODO: To limit the impact of the outpainting glitches, blend between sequences using this https://unix.stackexchange.com/a/500161\n",
    "\n",
    "    for i in reversed(range(min_dalle+1, num_dalle+1)):\n",
    "        print(i, end=': ')\n",
    "        animate_between(i, delay=delay, n=n, final_size=final_size, mode=mode, fps=fps, frames=frames)\n",
    "    if mode == 'gif':\n",
    "        check_output(f'convert movie_*.gif {filename}.gif', shell=True, stderr=STDOUT)\n",
    "        check_output('rm movie_*.gif', shell=True, stderr=STDOUT)\n",
    "    elif mode == 'mp4':\n",
    "        with open('inputs.txt', 'w') as f:\n",
    "            for i in range(min_dalle+1, num_dalle+1):\n",
    "                f.write(f'file movie_{i:03d}.mp4\\n')\n",
    "        check_output(f'ffmpeg -y -f concat -i inputs.txt -vcodec copy -acodec copy {filename}.mp4', shell=True, stderr=STDOUT)\n",
    "        check_output('rm inputs.txt; rm movie_*.mp4', shell=True, stderr=STDOUT)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb1682fe",
   "metadata": {},
   "source": [
    "# Animating the results\n",
    " * The `animate_between` function above can be used to generate the frames between each dalle2 output.\n",
    " * The `generate_full_animation` function will iterate through all the dalle2 outputs and animate them, and then will combine them into a single animated gif or mp4.\n",
    "  * If you generate a gif, you can speed up the results by using `gifsicle` to delete frames (or by setting `frames` to a lower value beforehand). See [here](https://graphicdesign.stackexchange.com/a/20937) for an example of how to do this afterwards `gifsicle`.\n",
    "  * If you generate an mp4, you can speed up the results by increasing the `fps`. See [here](https://trac.ffmpeg.org/wiki/ChangingFrameRate) for an example of doing this afterwards.\n",
    "  * If you want a video that is both forwards and backwards (looping), you can run `ffmpeg -i input.mp4 -filter_complex \"[0:v]reverse,fifo[r];[0:v][r] concat=n=2:v=1 [v]\" -map \"[v]\" output.mp4`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 497,
   "id": "b29bb050",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-07-18T21:25:44.135765Z",
     "start_time": "2022-07-18T21:21:30.730762Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9: 47=2511, 46=2464, 45=2417, 44=2371, 43=2326, 42=2282, 41=2239, 40=2197, 39=2155, 38=2115, 37=2075, 36=2035, 35=1997, 34=1959, 33=1922, 32=1886, 31=1850, 30=1815, 29=1781, 28=1747, 27=1714, 26=1682, 25=1650, 24=1619, 23=1588, 22=1558, 21=1528, 20=1500, 19=1471, 18=1443, 17=1416, 16=1389, 15=1363, 14=1337, 13=1312, 12=1287, 11=1263, 10=1239, 9=1215, 8=1192, 7=1170, 6=1148, 5=1126, 4=1105, 3=1084, 2=1063, 1=1043, \n",
      "8: 47=2511, 46=2464, 45=2417, 44=2371, 43=2326, 42=2282, 41=2239, 40=2197, 39=2155, 38=2115, 37=2075, 36=2035, 35=1997, 34=1959, 33=1922, 32=1886, 31=1850, 30=1815, 29=1781, 28=1747, 27=1714, 26=1682, 25=1650, 24=1619, 23=1588, 22=1558, 21=1528, 20=1500, 19=1471, 18=1443, 17=1416, 16=1389, 15=1363, 14=1337, 13=1312, 12=1287, 11=1263, 10=1239, 9=1215, 8=1192, 7=1170, 6=1148, 5=1126, 4=1105, 3=1084, 2=1063, 1=1043, \n"
     ]
    }
   ],
   "source": [
    "generate_full_animation(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e890209b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}