UnicodeDecodeError: 'utf-8' codec can't decode byte 0xff in position 0: invalid start byte when I tried to encode('utf-8')

Question

When I tried to use .encode('utf-8'), a "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xff in position 0: invalid start byte" comes out.

I was trying to crawl some images, and there is a problem confused me. I used urllib.request.urlopen(url).read().decode('utf-8') to convert the html page for get images' url. Later I want to save them.

However, I put all codes about urlopen into a function, so I don't want to change them. Then I thought maybe I can use encode('utf-8') to convert them back. After that, I don't know why but the exception raised.

import urllib.request as ur
import os

def getresponse(url):
    head = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'}
    req = ur.Request(url, None, head)
    response = ur.urlopen(req)
    result = response.read().decode('utf-8')
    return result


def saveimg(urlimg):
    img = getresponse(urlimg).encode('utf-8')
    file = open('xx.jpg', 'wb')
    file.write(img)
    file.close()


def downloadimg(url):
    os.chdir('/Users/xxxxxxx/Desktop')
    os.mkdir('xx')
    os.chdir('xx')
    saveimg(url)

downloadimg('https://cn.bing.com/th?id=OHR.WaterperryGardens_ZH-CN5767279278_1920x1080.jpg&rf=LaDigue_1920x1080.jpg&pid=hp')

I just want to know the reason of this.

Answer 1

May I propose a solution with requests?

import os
import requests  # pip install requests

def download(url, headers, dest_folder, new_file_name):
    if not os.path.exists(dest_folder):
        os.makedirs(dest_folder)  # create folder if it does not exist

    filename = new_file_name.replace(" ", "_")  # be careful with file names
    file_path = os.path.join(dest_folder, filename)

    r = requests.get(url, headers=headers, stream=True)
    if r.ok:
        print("saving to", os.path.abspath(file_path))
        with open(file_path, 'wb') as f:
            for chunk in r.iter_content(chunk_size=1024 * 8):
                if chunk:
                    f.write(chunk)
                    f.flush()
                    os.fsync(f.fileno())
    else:  # HTTP status code 4XX/5XX
        print("Download failed: status code {}\n{}".format(r.status_code, r.text))


headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'}
url = 'https://cn.bing.com/th?id=OHR.WaterperryGardens_ZH-CN5767279278_1920x1080.jpg&rf=LaDigue_1920x1080.jpg&pid=hp'
dest_folder = '/Users/xxxxxxx/Desktop/yyyy'
new_file_name = 'xx.jpg'

# save file to /Users/xxxxxxx/Desktop/yyyy/xx.jpg
# appropriate folders will be created if they do not exist
download(url, headers, dest_folder, new_file_name)