Search code examples
phpmysqlsql-updatebulkupdate

Update large dataset


I can upload a csv file with 3000 records but when I try to update, it takes long time which leads to request timeout error.

INSERTION:

$sqlInsert = "INSERT ignore into datanyc (`Symbol`,`Data_date`,`Open`,`High`,`Low`,`Last`,`Volume`)
                values ('".$column0."','".$dataDate."','".$column[2]."','".$column[3]."',
                '".$column[4]."','".$column[5]."','".$column[6]."')";                       

UPDATE statement:

$date_query="SELECT max(Data_date) as Prev_date FROM datanyc WHERE Data_date < '$dataDate' ";
$date_result=mysqli_query($con,$date_query);
$fetch=mysqli_fetch_array($date_result);
$Prevdate=$fetch['Prev_date'];
$temp="SELECT Last as last, Symbol as symbol FROM datanyc WHERE Data_date = '$Prevdate'";
$date1=mysqli_query($con,$temp);
while($row=mysqli_fetch_array($date1))
{
    //array_push($symbol,$row['symbol']);
    //array_push($last,$row['last']);
    $qry="UPDATE datanyc SET Prevclose = '".$row['last']."' WHERE Symbol LIKE '".$row['symbol']."' AND Data_date= $dateDate";
    mysqli_query($con,$qry);
    //$date=$row['max(Data_date)'];
}

Solution

  • Firstly, add index on Symbol and Data_date columns,

    Secondly, change your update queries to one query, so you don't need to update in loop like this:

    $sql = "UPDATE
              datanyc AS a
            INNER JOIN datanyc b ON a.symbol = b.symbol
                         AND b.Data_date = '$Prevdate'
            SET a.Prevclose = b.last";
    $date1 = mysqli_query($con, $sql);