First post, so bear with me - I am reading from an Amazon-SQS queue using ReceiveMessage - Below is the code snippet. This all works fine (running about 3000 messages per-minute so far). However an lsof -p shows alot (hundreds at some point) of open file descriptors - is this expected ? or is there something that needs to be done to close un-used connections (if that's the correct terminology). Any advice would be appreciated.
for {
select {
default:
recvResult, rErr := c.AwsService.ReceiveMessage(&sqs.ReceiveMessageInput{
QueueUrl: c.AwsQueueURL.QueueUrl,
MaxNumberOfMessages: aws.Int64(c.AwsMaxMessages),
WaitTimeSeconds: aws.Int64(c.AwsLongPollTimeout),
VisibilityTimeout: aws.Int64(c.AwsVisibilityTimeout),
})
// error check
var wg sync.WaitGroup
msgCount := len(recvResult.Messages)
if msgCount > 0 {
for _, awsMsg := range recvResult.Messages {
wg.Add(1)
go func(m *sqs.Message) {
defer wg.Done()
// process message body, send results to another endpoint, not AWS
okToDelete := processAwsMessage(
workerId, c.NodeId, c.QueueId, c.SaveMessageOnError,
time.Now().UnixNano(), m.Body,
)
if okToDelete {
_, dErr := c.AwsService.DeleteMessage(
&sqs.DeleteMessageInput{
QueueUrl: c.AwsQueueURL.QueueUrl,
ReceiptHandle: m.ReceiptHandle,
})
// error check
}
}(awsMsg) // End go func
} end for messages
wg.Wait()
} // end msgCount > 0
//// Code to complete select (ctx.Done) and for, etc
you can control the number of connections aws.Config by setting MaxConnsPerHost
in your http client
config := &aws.Config{
Endpoint: aws.String("sqs.us-east-1.amazonaws.com"), // VPC endpoint here
Region: aws.String("us-east-1"),
HTTPClient: &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
MaxIdleConns: 380,
MaxIdleConnsPerHost: 160,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
},
},
}