Search code examples
javascriptpythonhtmlbounding-boxgradio

How to Load an Image onto a Canvas in Gradio with Custom HTML and JavaScript?

I am willing to make a bounding box editor implementation in gradio by creating html and js by myself (where user can move bboxes, scale and draw new ones). But i can't find a way to load an image to canvas. I would appreciate any kind of help

The best thing i've accomplished so far is loading an image to an image html element:

import gradio as gr
import numpy as np
import base64
from PIL import Image
from io import BytesIO

def image_to_base64(img):
    buffered = BytesIO()
    img.save(buffered, format="PNG")
    img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
    return img_str

def image_uploaded(img):
    img_pil = Image.fromarray(img)
    img_str = image_to_base64(img_pil)

    # HTML content with a canvas element
    html_content = f"""
    <html lang="en">
    <body>
        <img src="data:image/png;base64,{img_str}" alt="">
        <canvas id="canvas"></canvas>
    </body>
    </html>
    """
    
    return img, gr.HTML(html_content)

with gr.Blocks(js=None) as demo:
    uploaded_image = gr.Image(type="numpy", show_label=False, interactive=True, show_download_button=False)
    original_image = gr.Image(type="numpy", interactive=False, show_label=False, show_download_button=False)

    bounding_box_editor = gr.HTML()
    
    uploaded_image.upload(image_uploaded, inputs=uploaded_image, outputs=[original_image, bounding_box_editor])

demo.launch(share=False)

P.S. If there are other implementations to that - i would be happy to see them (where user can move bboxes, scale and draw new ones)

What I need help with is the JavaScript part to actually draw the uploaded image onto the canvas. Any guidance or examples would be greatly appreciated.

Solution

  • You can see it in many questions or tutorials

    You have to get canvas, next get it context .getContext('2d') and next put image with .drawImage(img, x, y, width, height)

    Something like this

    var image  = document.getElementById("image");
var canvas = document.getElementById("canvas");
var ctx = canvas.getContext('2d');
ctx.drawImage(image, 0, 0);

    But I found out that gr.HTML() can't run <script> and this makes problem how to run above code.

    But you may add script to gr.Block(js=script) with function which later you can use in onclick=".." assigned to button or in onload="..." assigned to image.

    Full code:

    import gradio as gr
import numpy as np
import base64
from PIL import Image
from io import BytesIO

def image_to_base64(img):
    buffered = BytesIO()
    img.save(buffered, format="PNG")
    img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
    return img_str

def image_uploaded(img):
    img_pil = Image.fromarray(img)
    img_str = image_to_base64(img_pil)

    img_str = f'data:image/png;base64,{img_str}'
    #img_str = f'https://upload.wikimedia.org/wikipedia/en/7/7d/Lenna_%28test_image%29.png'
    
    # HTML content with a canvas element
    html_content = f"""
<h1>Uploaded: img</h1>

<img id="image" src="{img_str}" alt="" onload="draw_image();">

<h1>Uploaded: canvas</h1>

<canvas id="canvas" width="600"></canvas>

<!-- <button onclick="draw_image();" style="background-color:#eee;padding:25px">Draw Image</button>  -->
"""

    return img, html_content

script = """
/* this function is executed automatically at start (not when you load canvas) */
/* so draw_image() has to be inside (not in place of `async ()`) */
async () => {
    globalThis.draw_image = () => {
        var image  = document.getElementById("image");
        var canvas = document.getElementById("canvas");
        var ctx = canvas.getContext('2d');
        ctx.drawImage(image, 0, 0);
    }
}
"""

with gr.Blocks(js=script) as demo:
    uploaded_image = gr.Image(type="numpy", show_label=False, interactive=True, show_download_button=False)
    original_image = gr.Image(type="numpy", interactive=False, show_label=False, show_download_button=False)

    bounding_box_editor = gr.HTML()
    
    uploaded_image.upload(image_uploaded, inputs=uploaded_image, outputs=[original_image, bounding_box_editor])
    
demo.launch(share=False)

    Source of information:

    Code which add second image (image Lenna from Wikipedia) and you can use buttons to select which image put on canvas.

    import gradio as gr
import numpy as np
import base64
from PIL import Image
from io import BytesIO

def image_to_base64(img):
    buffered = BytesIO()
    img.save(buffered, format="PNG")
    img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
    return img_str

def image_uploaded(img):
    img_pil = Image.fromarray(img)
    img_str = image_to_base64(img_pil)

    img_str_1 = f'data:image/png;base64,{img_str}'
    img_str_2 = f'https://upload.wikimedia.org/wikipedia/en/7/7d/Lenna_%28test_image%29.png'
    
    # HTML content with a canvas element
    html_content = f"""
<h1>Uploaded: img</h1>

<img id="image1" src="{img_str_1}" alt="" width="200" height="200"> <img id="image2" src="{img_str_2}" alt="" width="200" height="200">

<h1>Uploaded: canvas</h1>

<canvas id="canvas" width="200" height="200"></canvas>

<button onclick="draw_image('image1');" style="background-color:#eee;padding:25px">Draw Image 1</button>
<button onclick="draw_image('image2');" style="background-color:#eee;padding:25px">Draw Image 2</button>
"""

    return img, html_content

script = """
/* this function is executed automatically at start (not when you load canvas) */
/* so draw_image() has to be inside (not in place of `async ()`) */
async () => {
    globalThis.draw_image = (image_id) => {
        var image  = document.getElementById(image_id);
        var canvas = document.getElementById("canvas");
        var ctx = canvas.getContext('2d');
        ctx.drawImage(image, 0, 0, 200, 200);
    }
}
"""

with gr.Blocks(js=script) as demo:
    uploaded_image = gr.Image(type="numpy", show_label=False, interactive=True, show_download_button=False)
    original_image = gr.Image(type="numpy", interactive=False, show_label=False, show_download_button=False)

    bounding_box_editor = gr.HTML()
    
    uploaded_image.upload(image_uploaded, inputs=uploaded_image, outputs=[original_image, bounding_box_editor])
    
demo.launch(share=False)