Unable to send a local image that isn't under wwwroot to openai using its new .NET library

Question

I have an ASP.NET Core 6 Web API with the new official library from OpenAI (https://github.com/openai/openai-dotnet).

What I'm trying to do, is to use a local image file to OpenAI. The file isn't under wwwroot, but under backend/assets/1.jpg.

I've written a basic service to setup all the information needed in order to send a request to OpenAI. But the problem is that I'm not able to send the image.

I keep getting errors like "url is too long" or "invalid image",

Here is my code - OpenAiService:

using OpenAI.Chat;

namespace backend.Services
{
    public class OpenAiService
    {
        private readonly ChatClient _chatClient;
        private readonly ChatCompletionOptions _options;

        public OpenAiService(IConfiguration configuration)
        {
            var apiKey = configuration.GetValue<string>("OpenAI:Key");
            _chatClient = new ChatClient("gpt-4o", apiKey);

            _options = new ChatCompletionOptions()
            {
                MaxTokens = 300,
            };
        }

        public async Task<string> ExtractListOfItems()
        {
            var imagePath = Path.Combine(Directory.GetCurrentDirectory(), "Assets", "1.jpg");
            var localUrl = $"https://localhost:7068/assets/{Path.GetFileName(imagePath)}";
            
            var messages = new List<ChatMessage>
            {
                new UserChatMessage(new List<ChatMessageContentPart>
                {
                    ChatMessageContentPart.CreateTextMessageContentPart("Extract the items from the following image and return a list of items including prices and amount."),
                    ChatMessageContentPart.CreateImageMessageContentPart(new Uri(localUrl))
                })
            };

            var completion = await _chatClient.CompleteChatAsync(messages, _options);
            return completion.Value.ToString();
        }
    }
}

Demo controller for testing:

using Microsoft.AspNetCore.Mvc;
using System.Threading.Tasks;
using backend.Services;
using OpenAI;
using OpenAI.Chat;

namespace backend.Controllers;

[ApiController]
[Route("[controller]")]
public class OpenAiDemoController : ControllerBase
{
    private readonly OpenAiService _openAiService;

    public OpenAiDemoController(OpenAiService openAiService)
    {
        _openAiService = openAiService;
    }
    
    [HttpPost]
    [Route("extract-items")]
    public async Task<IActionResult> CompleteSentence()
    {
        var completion = await _openAiService.ExtractListOfItems();
        return Ok(completion);
    }
}

program.cs file:

using backend.Configurations;
using backend.Services;
using Microsoft.Extensions.FileProviders;

var builder = WebApplication.CreateBuilder(args);

// Add services to the container.

builder.Services.Configure<OpenAiConfig>(builder.Configuration.GetSection("OpenAI"));

//add services
builder.Services.AddSingleton<OpenAiService>();

builder.Services.AddControllers();
builder.Services.AddEndpointsApiExplorer();
builder.Services.AddSwaggerGen();

// builder.Services.AddScoped<IOpenAiService, OpenAiService>();

builder.Services.AddCors(opt =>
{
    opt.AddPolicy("AllowAll", builder =>
    {
        builder.AllowAnyOrigin()
            .AllowAnyMethod()
            .AllowAnyHeader();
    });
});

var app = builder.Build();

app.UseStaticFiles(new StaticFileOptions
{
    FileProvider = new PhysicalFileProvider(
        Path.Combine(builder.Environment.ContentRootPath, "Assets")),
    RequestPath = "/assets"
});

app.UseStaticFiles(); // This serves files from wwwroot

app.UseStaticFiles(new StaticFileOptions
{
    FileProvider = new PhysicalFileProvider(
        Path.Combine(builder.Environment.ContentRootPath, "Assets")),
    RequestPath = "/assets"
});

// Configure the HTTP request pipeline.
if (app.Environment.IsDevelopment())
{
    app.UseSwagger();
    app.UseSwaggerUI();
}

app.UseHttpsRedirection();
app.UseCors("AllowAll");
app.UseAuthorization();

app.MapControllers();

app.Run();

Any idea what I'm doing wrong?

Answer 1

I was able to solve it on my own. There's an example in openai official repo that helped me to solve it.

https://github.com/openai/openai-dotnet/blob/main/examples/Chat/Example05_ChatWithVisionAsync.cs

Tho, im not sure if my implementation is mostly correct. I will leave this open for any other suggestions.

service:

using OpenAI.Chat;


namespace backend.Services
{
    public class OpenAiService
    {
        private readonly ChatClient _chatClient;
        private readonly ChatCompletionOptions _options;

        public OpenAiService(IConfiguration configuration)
        {
            var apiKey = configuration.GetValue<string>("OpenAI:Key");
            _chatClient = new ChatClient("gpt-4o", apiKey);
            _options = new ChatCompletionOptions()
            {
                MaxTokens = 300,
            };
        }

        public async Task<string> ExtractListOfItems()
        {
            var imageFilePath = Path.Combine("Assets", "1.jpg");
            await using Stream imageStream = File.OpenRead(imageFilePath);
            var imageBytes = BinaryData.FromStream(imageStream);

            var messages = new List<ChatMessage>
            {
                new UserChatMessage(new List<ChatMessageContentPart>
                {
                    ChatMessageContentPart.CreateTextMessageContentPart("describe the image. "),
                    ChatMessageContentPart.CreateImageMessageContentPart(imageBytes, "image/png")
                })
            };

            var completion = await _chatClient.CompleteChatAsync(messages, _options);
            return completion.Value.ToString();
        }
        
    }
}

usage in controller:

using Microsoft.AspNetCore.Mvc;
using System.Threading.Tasks;
using backend.Services;
using OpenAI;
using OpenAI.Chat;

namespace backend.Controllers;

[ApiController]
[Route("[controller]")]
public class OpenAiDemoController : ControllerBase
{
    private readonly OpenAiService _openAiService;

    public OpenAiDemoController(OpenAiService openAiService)
    {
        _openAiService = openAiService;
    }
    
    [HttpPost]
    [Route("extract-items")]
    public async Task<IActionResult> CompleteSentence()
    {
        var completion = await _openAiService.ExtractListOfItems();
        return Ok(completion);
    }
    
}

no need to apply for static file middleware.