aws lambda duration spike on a specific time

Question

I have a main lambda function running which stores and retrieves data from aws RDS postgresql. 5 different lamba functions at different intervals calls this main lambda to store data to the RDS. Usually my main lambda function gets around 100~150K inovocations. but on particularly saturday lambda duration spikes to 4145ms(usually 500ms or less).

Some observations of mine.

1. Is it due to cold start? my init duration is average 700ms.
2. Is DB reading/writing data slowly? DB read operation average 5sec and writing 400ms. I am using Sequelize.
3. Is there any problem with my postgres config?

Request Code:

module.exports.postMatch = async (req, res) => {
  // console.log("data", req.body);
  if (req.body == null) {
    console.log("no match data");
    return res.json("no match data");
  } else {
    const Week = null;
    const Stage = null;
    const Round = null;
    const Season = null;
    const matches = req.body;
    const StaticID = matches["@static_id"];
    const MatchID = matches["@id"];
    let date = req.body["@formatted_date"];
    date = date.split(".").reverse().join(".");
    const time = req.body["@time"] === "TBA" ? "00:00" : req.body["@time"];
    const MatchDateString = moment(
      `${date} ${time}`,
      "YYYY-MM-DD HH:mm:ss"
    ).format();
    const MatchDate = moment(MatchDateString, "YYYY-MM-DD HH:mm:ss");
    const Status = matches["@status"];
    const TournamentID = matches.id;
    const TournamentName = matches.leagueNameOnly;
    const Team1Name = matches.localteam["@name"];
    const Team1ID = matches.localteam["@id"];
    const Team2Name = matches.visitorteam["@name"];

    const Team2ID = matches.visitorteam["@id"];
    const Team1Score = matches.localteam["@goals"];
    const Team2Score = matches.visitorteam["@goals"];
    const Team1PenaltyScore = matches.penalty
      ? matches.penalty["@localteam"]
      : null;
    const Team2PenaltyScore = matches.penalty
      ? matches.penalty["@visitorteam"]
      : null;
    //  [0-0] => ['0','-','0']
    const HalfTimeScore = matches.ht
      ? matches.ht["@score"]
          .replace("[", "")
          .replace("[", "")
          .replace(/\s/g, "")
          .trim()
          .split("")
      : null;
    //  [0-0] => ['0','-','0']
    const FullTimeScore = matches.ft
      ? matches.ft["@score"]
          .replace("[", "")
          .replace("[", "")
          .replace(/\s/g, "")
          .trim()
          .split("")
      : null;
    //  [0-0] => ['0','-','0']
    const ExtraTimeScore = matches.et
      ? matches.et["@score"]
          .replace("[", "")
          .replace("[", "")
          .replace(/\s/g, "")
          .trim()
          .split("")
      : null;

    const Team1HalfTimeScore = HalfTimeScore ? HalfTimeScore[0] : null;
    const Team2HalfTimeScore = HalfTimeScore ? HalfTimeScore[2] : null;

    const Team1FullTimeScore = FullTimeScore ? FullTimeScore[0] : null;
    const Team1ExtraTimeScore = ExtraTimeScore ? ExtraTimeScore[0] : null;

    const Team2FullTimeScore = FullTimeScore ? FullTimeScore[2] : null;
    const Team2ExtraTimeScore = ExtraTimeScore ? ExtraTimeScore[2] : null;
      
    const exisitng_match = await match.findOne({
      where: {
        StaticID: StaticID,
      },
    });
  
    // const exisitng_match = null;
    if (exisitng_match) {
      exisitng_match.MatchDate = MatchDate;
      exisitng_match.Status = matches["@status"];
      exisitng_match.Team1Score = matches.localteam["@goals"];
      exisitng_match.Team1FullTimeScore = Team1FullTimeScore;
      exisitng_match.Team1ExtraTimeScore = Team1ExtraTimeScore;
      exisitng_match.Team1HalfTimeScore = Team1HalfTimeScore;
      exisitng_match.Team1PenaltyScore = Team1PenaltyScore;

      exisitng_match.Team2Score = matches.visitorteam["@goals"];
      exisitng_match.Team2FullTimeScore = Team2FullTimeScore;
      exisitng_match.Team2ExtraTimeScore = Team2ExtraTimeScore;
      exisitng_match.Team2HalfTimeScore = Team2HalfTimeScore;
      exisitng_match.Team2PenaltyScore = Team2PenaltyScore;
      exisitng_match.updatedAt = new Date().toISOString();
      // console.log("exx")
      try {
        console.log("start",new Date().toISOString()," ",StaticID);
        await exisitng_match.save();
        console.log("end",new Date().toISOString()," ",StaticID);
        console.log("updated");
        return res.json("match updated");
        // console.log(Team1Name, " ", Team2Name, " ", "match updated");
      } catch (error) {
        console.log("match error", error.message);
      }
    } else {
      const postData = {
        MatchID,
        StaticID,
        Season,
        MatchDate,
        Status,
        Week,
        Stage,
        Round,
        TournamentID,
        TournamentName,
        Team1Name,
        Team1ID,
        Team2ID,
        Team2Name,
        Team1Score,
        Team1HalfTimeScore,
        Team1FullTimeScore,
        Team1PenaltyScore,
        Team1ExtraTimeScore,
        Team2Score,
        Team2HalfTimeScore,
        Team2FullTimeScore,
        Team2PenaltyScore,
        Team2ExtraTimeScore,
      };

      try {
        await match.create(postData);
        console.log("match created");
        return res.json("match created");
      } catch (error) {
        console.log("match error", error.message);
      }
    }
  }

  return res.json("done");
};

My Sequelize postgres Config:

 production: {
    username: "username",
    password: "password",
    database: "db-name",
    host: "rds-url",
    dialect: "postgres",
    port: 5432,
    dialectOptions: {
      connectTimeout: 1000000,
    },
    logging: false,
 },

Answer 1

It could be due to a high amount of connections. Have you considered using connection pooling with RDS Proxy? This could improve the RDS performance, because each invoked function can share the same DB connection instead of opening a new one every time.

To pinpoint the exact cause of the issue I encourage you to take a look at your RDS instance's metrics (including CPU usage, connections, and freeable memory) and see if they are spiking/dipping during the periods when you are experiencing high latency.