I'm trying to improve a function to build a Zip by adding goroutines to handle each file that has to be archived.
But it ends up with a panic
panic: runtime error: slice bounds out of range [4126:4096]
The target directory contains 190 files (500 Mo). I don't really understand what's wrong, Thank you in advance for your help
The function :
func BuildArchive() error {
var files []string
err := filepath.Walk("/tmp/dir-to-zip", func(filePath string, info os.FileInfo, err error) error {
if info.IsDir() {
return nil
}
if err != nil {
fmt.Println(err)
return err
}
files = append(files, filePath)
return nil
})
if err != nil {
return err
}
bundle, err := os.Create("/tmp/archive.zip")
if err != nil {
return err
}
bundleWriter := zip.NewWriter(bundle)
var wg sync.WaitGroup
wg.Add(len(files))
for _, filePath := range files {
go func(filePath string) {
defer wg.Done()
relPath := strings.TrimPrefix(filePath, fmt.Sprintf("%v/", filepath.Dir("/tmp/dir-to-zip")))
bundleFile, err := bundleWriter.Create(relPath)
if err != nil {
fmt.Println(err)
}
fsFile, err := os.Open(filePath)
if err != nil {
fmt.Println(err)
}
_, err = io.Copy(bundleFile, fsFile)
if err != nil {
fmt.Println(err)
}
}(filePath)
}
wg.Wait()
err = bundleWriter.Close()
if err != nil {
return err
}
return nil
}
The error occurs here :
_, err = io.Copy(bundleFile, fsFile)
if err != nil {
fmt.Println(err)
}
The stack trace :
goroutine 48 [running]:
bufio.(*Writer).Write(0xc00002a100, {0xc00041a400?, 0x3d?, 0xc00041a400?})
/usr/local/go/src/bufio/bufio.go:670 +0x1c8
archive/zip.(*countWriter).Write(0xc00000c138, {0xc00041a400?, 0x3d?, 0x4afa20?})
/usr/local/go/src/archive/zip/writer.go:601 +0x2e
io.WriteString({0x4e7538, 0xc00000c138}, {0xc0000212c9, 0x3d})
/usr/local/go/src/io/io.go:314 +0x91
archive/zip.writeHeader({0x4e7538, 0xc00000c138}, 0xc000220090)
/usr/local/go/src/archive/zip/writer.go:422 +0x5ec
archive/zip.(*Writer).CreateHeader(0xc0000760a0, 0xc00021e1b0)
/usr/local/go/src/archive/zip/writer.go:378 +0x797
archive/zip.(*Writer).Create(0x4e7698?, {0xc0000212c9, 0x3d})
/usr/local/go/src/archive/zip/writer.go:223 +0x6c
main.BuildArchive.func2({0xc0000212c0, 0x46})
/home/simba/go/src/foobar/main.go:79 +0x1c5
created by main.BuildArchive
/home/simba/go/src/foobar/main.go:73 +0x5aa
panic: runtime error: slice bounds out of range [:4126] with capacity 4096
goroutine 6 [running]:
bufio.(*Writer).Flush(0xc00002a100)
/usr/local/go/src/bufio/bufio.go:634 +0x171
bufio.(*Writer).Write(0xc00002a100, {0xc0001b4200?, 0xc000199b20?, 0xc000199b20?})
/usr/local/go/src/bufio/bufio.go:672 +0xd8
archive/zip.(*countWriter).Write(0xc00000c138, {0xc0001b4200?, 0x0?, 0xc000199b40?})
/usr/local/go/src/archive/zip/writer.go:601 +0x2e
archive/zip.(*countWriter).Write(0xc000220018, {0xc0001b4200?, 0xc0001b02c0?, 0xc0001b02f0?})
/usr/local/go/src/archive/zip/writer.go:601 +0x2e
compress/flate.(*huffmanBitWriter).write(...)
/usr/local/go/src/compress/flate/huffman_bit_writer.go:136
compress/flate.(*huffmanBitWriter).writeCode(0xc0001b41e0?, {0x6000?, 0x22?})
/usr/local/go/src/compress/flate/huffman_bit_writer.go:347 +0xe5
compress/flate.(*huffmanBitWriter).writeTokens(0xc0001b41e0, {0xc002558000, 0x4001, 0x403f800000403f?}, {0xc0001aa900, 0x11e, 0x108129000000000f?}, {0xc0001ac100, 0x1e, 0x1e})
/usr/local/go/src/compress/flate/huffman_bit_writer.go:583 +0xb9
compress/flate.(*huffmanBitWriter).writeBlock(0xc0001b41e0, {0xc002558000?, 0x20?, 0xd79?}, 0x0, {0x0, 0x0, 0x0})
/usr/local/go/src/compress/flate/huffman_bit_writer.go:495 +0x490
compress/flate.(*compressor).writeBlock(0xc0005a2000, {0xc002558000?, 0xc000032f00?, 0xc000199d28?}, 0x47739b?)
/usr/local/go/src/compress/flate/deflate.go:170 +0x9c
compress/flate.(*compressor).deflate(0xc0005a2000)
/usr/local/go/src/compress/flate/deflate.go:509 +0x59b
compress/flate.(*compressor).write(0xc0005a2000, {0xc00256a000?, 0x8000, 0xf311b6fd?})
/usr/local/go/src/compress/flate/deflate.go:554 +0x82
compress/flate.(*Writer).Write(...)
/usr/local/go/src/compress/flate/deflate.go:712
archive/zip.(*pooledFlateWriter).Write(0xc00020c040?, {0xc00256a000?, 0x8000?, 0x4af140?})
/usr/local/go/src/archive/zip/register.go:51 +0xc5
archive/zip.(*countWriter).Write(...)
/usr/local/go/src/archive/zip/writer.go:601
archive/zip.(*fileWriter).Write(0xc000222000, {0xc00256a000, 0x8000, 0x8000})
/usr/local/go/src/archive/zip/writer.go:533 +0x97
io.copyBuffer({0x4e7558, 0xc000222000}, {0x4e7678, 0xc0001f8008}, {0x0, 0x0, 0x0})
/usr/local/go/src/io/io.go:428 +0x204
io.Copy(...)
/usr/local/go/src/io/io.go:385
main.BuildArchive.func2({0xc00001c0c0, 0x35})
/home/simba/go/src/foobar/main.go:89 +0x385
created by main.BuildArchive
/home/simba/go/src/foobar/main.go:73 +0x5aa
exit status 2
zip.Writer is not safe for concurrent use. You launch multiple goroutines, each creating and writing to zip entries (files).
Writer.Create() documents that:
Create adds a file to the zip file using the provided name. It returns a Writer to which the file contents should be written.
[...] The file's contents must be written to the io.Writer before the next call to Create, CreateHeader, or Close.
You can't create a zip concurrently. io.Writers of each zip entry write to the same underlying file (or to the same io.Writer in general), even if your app doesn't panic or crash, the resulting zip archive will likely be invalid.