Implement call retries with httr::RETRY() function in API call (R)

Question

I use the UN Comtrade data API with R.

library(rjson)

get.Comtrade <- function(url="http://comtrade.un.org/api/get?"
                         ,maxrec=50000
                         ,type="C"
                         ,freq="A"
                         ,px="HS"
                         ,ps="now"
                         ,r
                         ,p
                         ,rg="all"
                         ,cc="TOTAL"
                         ,fmt="json"
)
{
  string<- paste(url
                 ,"max=",maxrec,"&" #maximum no. of records returned
                 ,"type=",type,"&" #type of trade (c=commodities)
                 ,"freq=",freq,"&" #frequency
                 ,"px=",px,"&" #classification
                 ,"ps=",ps,"&" #time period
                 ,"r=",r,"&" #reporting area
                 ,"p=",p,"&" #partner country
                 ,"rg=",rg,"&" #trade flow
                 ,"cc=",cc,"&" #classification code
                 ,"fmt=",fmt        #Format
                 ,sep = ""
  )
  
  if(fmt == "csv") {
    raw.data<- read.csv(string,header=TRUE)
    return(list(validation=NULL, data=raw.data))
  } else {
    if(fmt == "json" ) {
      raw.data<- fromJSON(file=string)
      data<- raw.data$dataset
      validation<- unlist(raw.data$validation, recursive=TRUE)
      ndata<- NULL
      if(length(data)> 0) {
        var.names<- names(data[[1]])
        data<- as.data.frame(t( sapply(data,rbind)))
        ndata<- NULL
        for(i in 1:ncol(data)){
          data[sapply(data[,i],is.null),i]<- NA
          ndata<- cbind(ndata, unlist(data[,i]))
        }
        ndata<- as.data.frame(ndata)
        colnames(ndata)<- var.names
      }
      return(list(validation=validation,data =ndata))
    }
  }
}

However, sometimes it fails to connect server and I need to run the code several times to start working. Solution given here, to use Retry() function, which retries a request until it succeeds, seems attractive. However, I have some difficulties implementing this function in the code given above. has anybody used it before and knows how to recode it?

Answer 1

An API call using httr::RETRY could look like the following:

library(httr)
library(jsonlite)

res <- RETRY(
    verb = "GET",
    url = "http://comtrade.un.org/",
    path = "api/get",
    encode = "json",
    times = 3,
    query = list(
      max = 50000,
      type = "C",
      freq = "A",
      px = "HS",
      ps = "now",
      r = 842,
      p = "124,484",
      rg = "all",
      cc = "TOTAL",
      fmt = "json"
    )
  )

# alternativ: returns dataset as a `list`:
# parsed_content <- content(res, as = "parsed")    

# returns dataset as a `data.frame`:
json_content <- content(res, as = "text")
parsed_content <- parse_json(json_content, simplifyVector = TRUE)
parsed_content$validation
parsed_content$dataset

I'd suggest rewriting the get.Comtrade function using httr:

get.Comtrade <- function(verb = "GET",
                         url = "http://comtrade.un.org/",
                         path = "api/get",
                         encode = "json",
                         times = 3,
                         max = 50000,
                         type = "C",
                         freq = "A",
                         px = "HS",
                         ps = "now",
                         r,
                         p,
                         rg = "all",
                         cc = "TOTAL",
                         fmt = "json") {
  res <- httr::RETRY(
    verb = verb,
    url = url,
    path = path,
    encode = encode,
    times = times,
    query = list(
      max = max,
      type = type,
      freq = freq,
      px = px,
      ps = ps,
      r = r,
      p = p,
      rg = rg,
      cc = cc,
      fmt = fmt
    )
  )
  jsonlite::parse_json(content(res, as = "text"), simplifyVector = TRUE)
}

s1 <- get.Comtrade(r = "842", p = "124,484", times = 5)
print(s1)

Please see this and this for more information on library(httr).