Azure Storage gives "Server failed to authenticate the request" error when queueing a base64 encoded message

I want to enqueue a message to Azure Storage that should then be picked up by a WebJob. I am making an API call (explanation why I don't use QueueClient below), and when I add base64 encoding to the message I get the error "Server failed to authenticate the request". I know that my authentication is fine because it works without the encoding. I have tried changing the content type from "text/plain" to "application/xml" as suggested somewhere, but that did nothing.

An explanation why I'm doing what I'm doing:

Unfortunately due to some assembly conflicts I cannot use the Azure.Storage.Queues and Azure.Storage.Blobs packages in my project. Because of this, I use Microsoft.Azure.Storage.Blob (I know it's deprecated but it's my only viable alternative), and for queues I make a direct API call to Queue Service (nuget doesn't offer an alternative package like it does for blobs).

If I just post the message to the Queue Service, this message will not be base64 encoded (since the QueueClient usually handles this), and because of this the QueueTrigger will not pick up these messages.

Here's my code:

var Client = new HttpClient();

            var StorageKey = System.Configuration.ConfigurationManager.ConnectionStrings["AzureStorageKey"].ConnectionString;
            var urlPath = string.Format("{0}/messages", Constants.AzureStorage.QueueName);
            var uri = new Uri(string.Format("https://{0}.queue.core.windows.net/{1}", Constants.AzureStorage.StorageAccount, urlPath));


                        string queue_message = $"<QueueMessage><MessageText>{Convert.ToBase64String(Encoding.UTF8.GetBytes(message))}</MessageText></QueueMessage>";


            string content_type = "text/plain";
            var RequestDateString = DateTime.UtcNow.ToString("R", CultureInfo.InvariantCulture);

            string StringToSign = String.Format("POST\n"
                + "\n" // content encoding
                + "\n" // content language
                + queue_message.Length + "\n" // content length
                + "\n" // content md5
                + content_type + "\n" // content type
                + "\n" // date
                + "\n" // if modified since
                + "\n" // if match
                + "\n" // if none match
                + "\n" // if unmodified since
                + "\n" // range
                + "x-ms-date:" + RequestDateString + "\nx-ms-version:" + Constants.AzureStorage.StorageServiceVersion + "\n" // headers
                + "/{0}/{1}/{2}", Constants.AzureStorage.StorageAccount, Constants.AzureStorage.QueueName, "messages"); //url

            string auth = SignThis(StringToSign, StorageKey, Constants.AzureStorage.StorageAccount);

            if (Client.DefaultRequestHeaders.Contains("Authorization"))
                Client.DefaultRequestHeaders.Remove("Authorization");

            Client.DefaultRequestHeaders.Add("Authorization", auth);
            Client.DefaultRequestHeaders.Accept.Clear();

            Client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(content_type));
            
            if (Client.DefaultRequestHeaders.Contains("x-ms-version"))
                Client.DefaultRequestHeaders.Remove("x-ms-version");
            Client.DefaultRequestHeaders.Add("x-ms-version", Constants.AzureStorage.StorageServiceVersion);

            if (Client.DefaultRequestHeaders.Contains("x-ms-date"))
                Client.DefaultRequestHeaders.Remove("x-ms-date");
            Client.DefaultRequestHeaders.Add("x-ms-date", RequestDateString);
            
            try
            {
                var stringContent = new StringContent(queue_message, Encoding.UTF8);
                var response = Client.PostAsync(uri, stringContent);

                var res = response.Result;
            }
            catch (Exception ex)
            {
                logger.Error("An error occured while creating a queue message", ex);
                throw ex;
            }

private static string SignThis(string StringToSign, string Key, string Account)
    {
        string signature = string.Empty;
        byte[] unicodeKey = Convert.FromBase64String(Key);
        using (HMACSHA256 hmacSha256 = new HMACSHA256(unicodeKey))
        {
            byte[] dataToHmac = Encoding.UTF8.GetBytes(StringToSign);
            signature = Convert.ToBase64String(hmacSha256.ComputeHash(dataToHmac));
        }

        string authorizationHeader = string.Format(
              CultureInfo.InvariantCulture,
              "{0} {1}:{2}",
              "SharedKey",
              Account,
              signature);

        return authorizationHeader;
    }

This is the complete response object I am getting:

    {StatusCode: 403, ReasonPhrase: 'Server failed to authenticate the request. Make sure the value of Authorization header is formed correctly including the signature.', Version: 1.1, Content: 
System.Net.Http.StreamContent, Headers:
{
  x-ms-request-id: 0f1be2b5-0003-0009-2960-e5e177000000
  x-ms-error-code: AuthenticationFailed
  Date: Fri, 21 Oct 2022 15:19:22 GMT
  Server: Microsoft-HTTPAPI/2.0
  Content-Length: 710
  Content-Type: application/xml
}}
    Content: {System.Net.Http.StreamContent}
    Headers: {x-ms-request-id: 0f1be2b5-0003-0009-2960-e5e177000000
x-ms-error-code: AuthenticationFailed
Date: Fri, 21 Oct 2022 15:19:22 GMT
Server: Microsoft-HTTPAPI/2.0
}
    IsSuccessStatusCode: false
    ReasonPhrase: "Server failed to authenticate the request. Make sure the value of Authorization header is formed correctly including the signature."
    RequestMessage: {Method: PUT, RequestUri: 'https://mystorageaccount.queue.core.windows.net/myqueuename/messages', Version: 1.1, Content: System.Net.Http.StringContent, Headers:
{
  Authorization: SharedKey efrelsblobstorage:*****
  Accept: application/xml
  x-ms-version: 2021-08-06
  x-ms-date: Fri, 21 Oct 2022 15:19:26 GMT
  Content-Type: application/xml; charset=utf-8
  Content-Length: 212
}}
    StatusCode: Forbidden
    Version: {1.1}
    StatusCode: Forbidden
    Version: {1.1}

Solution

A few issues with your code:

You're base64 encoding the entire message string including <QueueMessage><MessageText>. Only thing you should be base64 encoding is the actual message content (your message variable).
The reason you are getting 403 error message is because the value of Content-Length header is mismatched. In your StringToSign you're passing the length as the length of queue_message variable however the content length of your request body is based on the base64 encoded value of this value.

To fix this, please change the following line of code:

string queue_message = $"<QueueMessage><MessageText>{message}</MessageText></QueueMessage>";

string queue_message = $"<QueueMessage><MessageText>{Convert.ToBase64String(Encoding.UTF8.GetBytes(message))}</MessageText></QueueMessage>";

and then change the following line of code:

var stringContent = new StringContent(encodedMessage, Encoding.UTF8);

var stringContent = new StringContent(queue_message, Encoding.UTF8);

You can also get rid of the following line of code:

var encodedMessage = Base64Encode(queue_message);

UPDATE

One more reason your request could be failing is because of reuqest method mismatch. Put Message request is an HTTP POST operation however you are calling an HTTP PUT.

Please change the following line of code:

var response = Client.PutAsync(uri, stringContent);

var response = Client.PostAsync(uri, stringContent);

and that should fix the error.

UPDATE 2

This time the issue was with the content type request header. In your StringToSign the value was set as text/plain however the value sent in the request was text/plain; charset=utf-8.

To fix the issue, please make the following 2 changes:

Change

string content_type = "text/plain";

string content_type = "text/plain; charset=utf-8";

and get rid of the following line of code:

Client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(content_type));

after that your code should work.

Here's the code that I wrote which works:

using System.Globalization;
using System.Net.Http.Headers;
using System.Security.Cryptography;
using System.Text;

static string SignThis(string StringToSign, string Key, string Account)
{
    string signature = string.Empty;
    byte[] unicodeKey = Convert.FromBase64String(Key);
    using (HMACSHA256 hmacSha256 = new HMACSHA256(unicodeKey))
    {
        byte[] dataToHmac = Encoding.UTF8.GetBytes(StringToSign);
        signature = Convert.ToBase64String(hmacSha256.ComputeHash(dataToHmac));
    }

    string authorizationHeader = string.Format(
        CultureInfo.InvariantCulture,
        "{0} {1}:{2}",
        "SharedKey",
        Account,
        signature);

    return authorizationHeader;
}

var Client = new HttpClient();
var StorageServiceVersion = "2021-08-06";
var StorageAccount = "<my-account-name>";
var StorageKey = "<my-account-key>";
var QueueName = "<my-queue-name>";
var message = "Hello World!";

var urlPath = string.Format("{0}/messages", QueueName);
var uri = new Uri(string.Format("https://{0}.queue.core.windows.net/{1}", StorageAccount, urlPath));

string queue_message = $"<QueueMessage><MessageText>{Convert.ToBase64String(Encoding.UTF8.GetBytes(message))}</MessageText></QueueMessage>";

string content_type = "text/plain; charset=utf-8";

var RequestDateString = DateTime.UtcNow.ToString("R", CultureInfo.InvariantCulture);

string StringToSign = String.Format("POST\n"
    + "\n" // content encoding
    + "\n" // content language
    + queue_message.Length + "\n" // content length
    + "\n" // content md5
    + content_type + "\n" // content type
    + "\n" // date
    + "\n" // if modified since
    + "\n" // if match
    + "\n" // if none match
    + "\n" // if unmodified since
    + "\n" // range
    + "x-ms-date:" + RequestDateString + "\nx-ms-version:" + StorageServiceVersion + "\n" // headers
    + "/{0}/{1}/{2}", StorageAccount, QueueName, "messages"); //url

string auth = SignThis(StringToSign, StorageKey, StorageAccount);

if (Client.DefaultRequestHeaders.Contains("Authorization"))
    Client.DefaultRequestHeaders.Remove("Authorization");

Client.DefaultRequestHeaders.Add("Authorization", auth);
Client.DefaultRequestHeaders.Accept.Clear();

if (Client.DefaultRequestHeaders.Contains("x-ms-version"))
    Client.DefaultRequestHeaders.Remove("x-ms-version");
Client.DefaultRequestHeaders.Add("x-ms-version", StorageServiceVersion);

if (Client.DefaultRequestHeaders.Contains("x-ms-date"))
    Client.DefaultRequestHeaders.Remove("x-ms-date");
Client.DefaultRequestHeaders.Add("x-ms-date", RequestDateString);

try
{
    var stringContent = new StringContent(queue_message, Encoding.UTF8);
    var response = Client.PostAsync(uri, stringContent);

    var res = response.Result;
    var responseData = await res.Content.ReadAsStringAsync();
}
catch (Exception ex)
{
    Console.WriteLine("An error occured while creating a queue message", ex);
    throw ex;
}