Golang function generalization for different packages

Question

Imagine these functions needs to be used how can I make this calls generic so that I don't repeat almost the same code.

with "encoding/csv"

func getDataFromCSVFiles(files []string) (error, Data) {
        data := Data{}
        for _, file := range files {
            f, err := os.Open(file)
            if err != nil {
                panic(err)
                return err, data
            }
            defer f.Close()
            r := charmap.ISO8859_1.NewDecoder().Reader(f)
            reader := csv.NewReader(r)
            for i := 1;;i++ {
                rec, err := reader.Read()
                if i == 1 {
                    //Skipping header
                    continue
                }
                if err != nil {
                    if err == io.EOF {
                        break
                    }
                    //TODO log error line and csv filename
                    log.Fatal(err)
                }
                addWorkbook(rec, &data)
            }
        }
        return nil, data
    }

and with import fw "github.com/hduplooy/gofixedwidth" which is almost the same except calling fw.NewReader

func getDataFromPRNFiles(files []string) (error, Data) {
    data := Data{}
    for _, file := range files {
        f, err := os.Open(file)
        if err != nil {
            panic(err)
            return err, data
        }
        defer f.Close()
        r := charmap.ISO8859_1.NewDecoder().Reader(f)
        reader := fw.NewReader(r)
        for i := 1;;i++ {
            rec, err := reader.Read()
            if i == 1 {
                //Skipping header
                continue
            }
            if err != nil {
                if err == io.EOF {
                    break
                }
                //TODO log error line and csv filename
                log.Fatal(err)
            }
            addWorkbook(rec, &data)
        }
    }
    return nil, data
}

Answer 1

The only apparent difference is:

reader := csv.NewReader(r)

versus:

reader := fw.NewReader(r)

I'm not sure what fw is but presumably both readers implement a common interface:

type StringSliceReader interface {
    Read() ([]string, error)
}

So you could pass the openers (csv.NewReader and fw.NewReader) as function arguments:

func getDataFromFiles(files []string, func(r io.Reader) StringArrayReader) (error, Data) {
    //...
}

but you'd need to wrap them in little functions to get around the return types:

func newCSVReader(r io.Reader) StringSliceReader {
    return csv.NewReader(r)
}

func newFWReader(r io.Reader) StringSliceReader {
    return fw.NewReader(r)
}

Also, defer queues up things to execute when the function exits, not on the next iteration of a loop. So if you do this:

for _, file := range files {
    f, err := os.Open(file)
    if err != nil {
        panic(err)
        return err, data
    }
    defer f.Close()
    //...
}

and files has a hundred entries then you'll have a hundred open files before any of them are closed. You probably want to move that loop body to a separate function so that you only have one file open at a time.

Furthermore, error is usually the last return value from a function so you should return data, err to be more idiomatic.

The result could look something like this:

type StringSliceReader interface {
    Read() ([]string, error)
}

type NewReader func(r io.Reader) StringSliceReader

func newCSVReader(r io.Reader) StringSliceReader {
    return csv.NewReader(r)
}

func newFWReader(r io.Reader) StringSliceReader {
    return fw.NewReader(r)
}

func getDataFrom(file string, data *Data, newReader NewReader) error {
    f, err := os.Open(file)
    if err != nil {
        return err
    }
    defer f.Close()

    r := charmap.ISO8859_1.NewDecoder().Reader(f)
    reader := newReader(r)
    for i := 1; ; i++ {
        rec, err := reader.Read()
        if i == 1 {
            continue
        }
        if err != nil {
            if err == io.EOF {
                break
            }
            log.Fatal(err)
        }
        addWorkbook(rec, data)
    }
    return nil
}

func getDataFromFiles(files []string, newReader NewReader) (Data, error) {
    data := Data{}
    for _, file := range files {
        err := getDataFrom(file, newReader, &data)
        if err != nil {
            panic(err)
            return data, err
        }
    }
    return data, nil
}

and you could say getDataFromFiles(files, newCSVReader) to read CSVs or getDataFromFiles(files, newFWReader) to read FW files. If you want to read from something else, you'd just need a NewReader function and something that implements the StringSliceReader interface.

You might want to bury/hide the charmap.ISO8859_1.NewDecoder().Reader(f) stuff inside the NewReader functions to make it easier to read non-Latin-1 encoded files. You could also replace newReader NewReader with a map[string]NewReader in getDataFromFiles and choose the NewReader to use based on the file's extension or other format identifier.