Search code examples
jsongogo-colly

I cannot print data side by side in JSON array in Golang Colly


I'm taking pictures with golang colly from Amazon and I want to throw these pictures in JSON into a single array (only 1 array for each product images). I scraped the pictures I need, I'm just having a problem with the JSON file. Thank you very much in advance.

package main

import (
    "encoding/json"
    "fmt"
    "io/ioutil"
    "log"
    "os"
    "time"

    "github.com/gocolly/colly"
)

type Info struct {
    ID     int      `json:"id"`
    Images []string `json:"images"`
}

func main() {
    AllInfos := make([]Info, 0)

    start := time.Now()

    co := colly.NewCollector(
        colly.AllowedDomains("www.amazon.com", "amazon.com"),
    )

    // GET Images
    Counter := 0
    var info Info
    var theArray [10]string
    co.OnHTML("img[src]", func(e *colly.HTMLElement) {
        imgsrc := e.Attr("src")
        imgclass := e.Attr("class")

        if imgsrc[0:49] == "https://images-na.ssl-images-amazon.com/images/I/" && imgclass == "" {
            theArray[Counter] = imgsrc
            fmt.Printf("The Array %d %v", Counter, theArray[Counter]+"\n")
            Counter = Counter + 1
            co.Visit(e.Request.AbsoluteURL(imgsrc))

            info = Info{
                Images: []string{
                    theArray[0],
                    theArray[1],
                    theArray[2],
                    theArray[3],
                    theArray[4],
                    theArray[5],
                    theArray[6],
                },
            }
            AllInfos = append(AllInfos, info)
        }
    })

    co.Visit("https://www.amazon.com/Bluetooth-FM-Transmitter-Compatible-Smartphones/dp/B088TCSH8T/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=A2XMGHKVCX40WB&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFGVEgxTjJLRFdLSkUmZW5jcnlwdGVkSWQ9QTAyNDE0ODkyRlNDWlAzUktPQzJSJmVuY3J5cHRlZEFkSWQ9QTA5MTkwNjgyWVUzQ0RUMTBCTEFFJndpZGdldE5hbWU9c3BfYXRmJmFjdGlvbj1jbGlja1JlZGlyZWN0JmRvTm90TG9nQ2xpY2s9dHJ1ZQ==WDY4VktWQlImd2lkZ2V0TmFtZT1zcF9kZXRhaWwmYWN0aW9uPWNsaWNrUmVkaXJlY3QmZG9Ob3RMb2dDbGljaz10cnVl")

    co.OnRequest(func(r *colly.Request) {
        fmt.Println("Visiting: ", r.URL.String())
    })

    enc := json.NewEncoder(os.Stdout)
    enc.SetIndent("", " ")
    enc.Encode(AllInfos)

    writeJson(AllInfos)

    duration := time.Since(start)

    fmt.Println(duration.Seconds())
}

func writeJson(data []Info) {
    dataFile, err := json.MarshalIndent(data, "", " ")
    if err != nil {
        log.Println("Could not create JSON", err)
    }

    ioutil.WriteFile("stocky.json", dataFile, 0666)
}

My code's JSON result;

[
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "",
   "",
   "",
   "",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
   "",
   "",
   "",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
   "",
   "",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
   "",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51hwJpj4OgL._AC_US40_.jpg",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51hwJpj4OgL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51dz9PNWVrL._AC_US40_.jpg",
   ""
  ]
 }
]

I need JSON result like this;

[
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51hwJpj4OgL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51dz9PNWVrL._AC_US40_.jpg",
  ]
 }
]

Solution

  • I had a hard time understanding what you wanted to do but here's what I came up with:

    package main
    
    import (
        "encoding/json"
        "fmt"
        "io/ioutil"
        "log"
        "os"
        "strings"
        "time"
    
        "github.com/gocolly/colly"
    )
    
    type Info struct {
        ID     int      `json:"id"`
        Images []string `json:"images"`
    }
    
    func main() {
        allInfos := new(Info)
    
        start := time.Now()
    
        co := colly.NewCollector(
            colly.AllowedDomains("www.amazon.com", "amazon.com"),
        )
    
        // GET Images
        co.OnHTML("img[src]", func(e *colly.HTMLElement) {
            imgsrc := e.Attr("src")
            imgclass := e.Attr("class")
    
            if strings.HasPrefix(imgsrc, "https://images-na.ssl-images-amazon.com/images/I/") && imgclass == "" {
                fmt.Printf("The Array %d %v", len(allInfos.Images), imgsrc+"\n")
                allInfos.Images = append(allInfos.Images, imgsrc)
                co.Visit(e.Request.AbsoluteURL(imgsrc))
            }
        })
    
        co.Visit("https://www.amazon.com/Bluetooth-FM-Transmitter-Compatible-Smartphones/dp/B088TCSH8T/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=A2XMGHKVCX40WB&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFGVEgxTjJLRFdLSkUmZW5jcnlwdGVkSWQ9QTAyNDE0ODkyRlNDWlAzUktPQzJSJmVuY3J5cHRlZEFkSWQ9QTA5MTkwNjgyWVUzQ0RUMTBCTEFFJndpZGdldE5hbWU9c3BfYXRmJmFjdGlvbj1jbGlja1JlZGlyZWN0JmRvTm90TG9nQ2xpY2s9dHJ1ZQ==WDY4VktWQlImd2lkZ2V0TmFtZT1zcF9kZXRhaWwmYWN0aW9uPWNsaWNrUmVkaXJlY3QmZG9Ob3RMb2dDbGljaz10cnVl")
    
        co.OnRequest(func(r *colly.Request) {
            fmt.Println("Visiting: ", r.URL.String())
        })
    
        enc := json.NewEncoder(os.Stdout)
        enc.SetIndent("", " ")
        enc.Encode(allInfos)
    
        writeJson(allInfos)
    
        duration := time.Since(start)
    
        fmt.Println(duration.Seconds())
    }
    
    func writeJson(data *Info) {
        dataFile, err := json.MarshalIndent(data, "", " ")
        if err != nil {
            log.Println("Could not create JSON", err)
        }
    
        ioutil.WriteFile("stocky.json", dataFile, 0666)
    }
    

    As @Flimzy suggested, I appended only the new imagesrc to a since instance of an info struct. Seeing as it seemed like your expected output was a slice of length 1 with all the images.

    I also used a strings.HasPrefix to clean up your slice check. This is equivalent but I think it's cleaner, so that's an optional change.

    Let me know if you have any more questions.

    EDIT, if you need to keep using a slice something like the following might work. From your question it's hard to tell when you need to clear a new Info:

    package main
    
    import (
        "encoding/json"
        "fmt"
        "io/ioutil"
        "log"
        "os"
        "strings"
        "time"
    
        "github.com/gocolly/colly"
    )
    
    type Info struct {
        ID     int      `json:"id"`
        Images []string `json:"images"`
    }
    
    func main() {
        allInfos := make([]Info, 1)
    
        start := time.Now()
    
        co := colly.NewCollector(
            colly.AllowedDomains("www.amazon.com", "amazon.com"),
        )
    
        // GET Images
        co.OnHTML("img[src]", func(e *colly.HTMLElement) {
            imgsrc := e.Attr("src")
            imgclass := e.Attr("class")
    
            if strings.HasPrefix(imgsrc, "https://images-na.ssl-images-amazon.com/images/I/") && imgclass == "" {
                fmt.Printf("The Array %d %v", len(allInfos[0].Images), imgsrc+"\n")
                allInfos[0].Images = append(allInfos[0].Images, imgsrc)
                co.Visit(e.Request.AbsoluteURL(imgsrc))
            }
        })
    
        co.Visit("https://www.amazon.com/Bluetooth-FM-Transmitter-Compatible-Smartphones/dp/B088TCSH8T/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=A2XMGHKVCX40WB&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFGVEgxTjJLRFdLSkUmZW5jcnlwdGVkSWQ9QTAyNDE0ODkyRlNDWlAzUktPQzJSJmVuY3J5cHRlZEFkSWQ9QTA5MTkwNjgyWVUzQ0RUMTBCTEFFJndpZGdldE5hbWU9c3BfYXRmJmFjdGlvbj1jbGlja1JlZGlyZWN0JmRvTm90TG9nQ2xpY2s9dHJ1ZQ==WDY4VktWQlImd2lkZ2V0TmFtZT1zcF9kZXRhaWwmYWN0aW9uPWNsaWNrUmVkaXJlY3QmZG9Ob3RMb2dDbGljaz10cnVl")
    
        co.OnRequest(func(r *colly.Request) {
            fmt.Println("Visiting: ", r.URL.String())
        })
    
        enc := json.NewEncoder(os.Stdout)
        enc.SetIndent("", " ")
        enc.Encode(allInfos)
    
        writeJson(allInfos)
    
        duration := time.Since(start)
    
        fmt.Println(duration.Seconds())
    }
    
    func writeJson(data []Info) {
        dataFile, err := json.MarshalIndent(data, "", " ")
        if err != nil {
            log.Println("Could not create JSON", err)
        }
    
        ioutil.WriteFile("stocky.json", dataFile, 0666)
    }