Why do I get 301 Moved Permanently

Question

Why does this code result in 301 error when trying to access sites that has a .net suffix

void test(const char * host, const char *index)
{
    BIO *bio, *out;
    SSL_CTX * ctx;
    SSL * ssl;
    int len;
    char tmpbuf[1024];
    ERR_load_crypto_strings();
    char ready[1204];
    char format[] = "%s:http";
    sprintf(ready, format , host);
    char req_template[] =  "GET %s HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n";
    char ready_request[1024];
    sprintf(ready_request , req_template , index, host);
    const SSL_METHOD * method = SSLv23_client_method(); 
    if (!method)
        exit(-1);
    ctx = SSL_CTX_new(method);
    if (!ctx)
        exit(-1);
    if (!SSL_CTX_load_verify_locations(ctx,"/etc/ssl/certs/ca-certificates.crt","/etc/ssl/certs/"))
    {
        SSL_CTX_free(ctx);
        exit(-1);
    }
    bio = BIO_new_ssl_connect(ctx); 
    BIO_get_ssl(bio, &ssl);
    SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);
    char temp[1024];
    sprintf(temp, "%s:https",host);
    if (BIO_set_conn_hostname(bio, temp) < 0)
    {
        memset(temp, 0, sizeof(temp));
        sprintf(temp, "%s:http", host);
        bio = BIO_new_connect(temp);
        if (BIO_do_connect(bio) < 0 )
        {
            BIO_free_all(bio);
            SSL_CTX_free(ctx);
            exit(-1);
        }       
    }
    printf("###\n%s\n###\n",ready_request);
    out = BIO_new_fp(stdout, BIO_NOCLOSE);
    if(BIO_do_connect(bio) <= 0) 
        exit(-1);

    BIO_puts(bio,ready_request);
    for(;;) 
    {
        len = BIO_read(bio, tmpbuf, 1024);
        if(len <= 0) break;
        BIO_write(out, tmpbuf, len);
    }
    BIO_free(bio);
    BIO_free(out);
}
int main()
{

    test("openssl.org", "/docs/manpages.html");
    test("pastebin.com", "/raw/j0BnRwBw");
    test("pastebin.com", "/j0BnRwBw");

}

for some reason that i can't figure out the first time test is called it returns a 301 status code but the two time test is called it returns the html code or the paste code with out any problems

Does this have anything to do with the fact that the websites use different technologies or if they have some sort of firewall, I believe pastebin uses cloudflare to protect it self, I also tried using User-Agent but still got the same result

Answer 1

Just add www. as a prefix to the host header and for BIO_set_conn_hostname you should use the format www.<hostname>.com:https or www.<host>.org:http for BIO_new_connect

for some reason, the docs do not mention this

void test(const char * host, const char *index)
{
    BIO *bio, *out;
    SSL_CTX * ctx;
    SSL * ssl;
    int len;
    char tmpbuf[1024];
    ERR_load_crypto_strings();
    char req_template[] =  "GET %s HTTP/1.1\r\nHost: www.%s\r\nConnection: close\r\n\r\n";
    char ready_request[1024];
    sprintf(ready_request , req_template , index, host);
    const SSL_METHOD * method = SSLv23_client_method(); 
    if (!method)
        exit(-1);
    ctx = SSL_CTX_new(method);
    if (!ctx)
        exit(-1);
    if (!SSL_CTX_load_verify_locations(ctx,"/etc/ssl/certs/ca-certificates.crt","/etc/ssl/certs/"))
    {
        SSL_CTX_free(ctx);
        exit(-1);
    }
    bio = BIO_new_ssl_connect(ctx); 
    BIO_get_ssl(bio, &ssl);
    SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);
    char temp[1024];
    sprintf(temp, "www.%s:https",host);
    if (BIO_set_conn_hostname(bio, temp) < 0)
    {
        memset(temp, 0, sizeof(temp));
        sprintf(temp, "www.%s:http", host);
        bio = BIO_new_connect(temp);
        if (BIO_do_connect(bio) < 0 )
        {
            BIO_free_all(bio);
            SSL_CTX_free(ctx);
            exit(-1);
        }       
    }
    printf("###\n%s\n###\n",ready_request);
    out = BIO_new_fp(stdout, BIO_NOCLOSE);
    if(BIO_do_connect(bio) <= 0) 
        exit(-1);

    BIO_puts(bio,ready_request);
    for(;;) 
    {
        len = BIO_read(bio, tmpbuf, 1024);
        if(len <= 0) break;
        BIO_write(out, tmpbuf, len);
    }
    BIO_free(bio);
    BIO_free(out);
}