Send os.Stdin via http POST without loading file into memory

Question

I am trying to send a file to a server via a POST request. To accomplish this, I use the following code:

func newfileUploadRequest(uri string) (*http.Request, error) {

    body := new(bytes.Buffer)
    writer := multipart.NewWriter(body)
    part, err := writer.CreateFormFile("file", "file")
    if err != nil {
        return nil, err
    }
    io.Copy(part, os.Stdin)

    err = writer.Close()
    if err != nil {
        return nil, err
    }

    request, err := http.NewRequest("POST", uri, body)
    if err != nil {
        return nil, err
    }
    request.Header.Set("Content-Type", writer.FormDataContentType())
    return request, nil
}


func main() {
    r, err := newfileUploadRequest("http://localhost:8080/")
    if err != nil {
        panic(err)
    }

    client := &http.Client{}
    resp, err := client.Do(r)
    if err != nil {
        panic(err)
    }
    body, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        panic(err)
    }
    print(string(body))
}

While this works well, it is my understanding that io.Copy will copy the entire file into memory before the POST request is sent. Large files (multiple GB) will create issues. Is there a way to prevent this? I found this, but that simply says to use io.Copy.

Answer 1

You can avoid copying the data in memory by using an io.Pipe and a goroutine that copies from the file to the pipe:

func newfileUploadRequest(uri string) (*http.Request, error) {
  r, w := io.Pipe()
  writer := multipart.NewWriter(w)
  go func() {
    part, err := writer.CreateFormFile("file", "file")
    if err != nil {
        w.CloseWithError(err)
        return
    }
    _, err = io.Copy(part, os.Stdin)
    if err != nil {
        w.CloseWithError(err)
        return
    }
    err = writer.Close()
    if err != nil {
        w.CloseWithError(err)
        return
    }
  }()

  request, err := http.NewRequest("POST", uri, r)
  if err != nil {
    return nil, err
  }
  request.Header.Set("Content-Type", writer.FormDataContentType())
  return request, nil
}