Edit
Just found the post plotting 50 million points with d3.js.
Sluggish interaction with zoom and pan are due to too many elements in the svg. The key is to use hierarchical levels of detail, just like the image pyramid. , to limit the maximum elements in svg.
Original post
I am trying to read some data points from csv/excel file and plot them using d3.js.
The data set contains 100,000s of rows, each row contains a time stamp and a value at that time.
Time stamp, pressure
12/17/2019 12:00:00 AM, 600
I followed this example to plot the time-pressure chart with zoom and pan.
There is no issue and worked perfectly.
One issue is that when working with large data set, say 500,000 of data points, the interaction with the chart is sluggish.
The chart with 500,000 data points shows an overall shape, and the details would only come up when zoomed in at large scale.
When zoomed in, all the data points are re-plotted and clipped out by the clip path. Would there be some room to improve the speed?
Updated Code
function draw(res){
//clear the current content in the div
document.getElementById("spectrum-fig").innerHTML = '';
var fullwidth = d3.select('#spectrum-fig').node().getBoundingClientRect().width;
fullwidth = fullwidth < 500? 500:fullwidth;
var fullheight = 500;
var resLevelOne = getWindowed(res, 1);
var resLevelTwo = getWindowed(res, 2);
var designMax= getMaxPressureKPa();
var resMax = getPsiTopTen(res);
const SMYSKPa = getSMYSPressureKPa();
const avePsi = getAvePsi(res);
var psiRange = d3.max(res, d=>d.psi) - d3.min(res, d=>d.psi);
var resSmallChart = getWindowed(res, 2);//
//filtSpectrum(res, 0.05*psiRange); //0.05 magic numbers
//var resSmallChart = res;
//margin for focus chart, margin for small chart
var margin = {left:50, right: 50, top: 30, bottom:170},
margin2 = {left:50, right: 50, top: 360, bottom:30},
width = fullwidth - margin.left - margin.right,
height = fullheight - margin.top - margin.bottom,
height2 = fullheight - margin2.top-margin2.bottom;
//x, y, for big chart, x2, y2 for small chart
var x = d3.scaleTime().domain(d3.extent(res, d => d.Time)).range([0, width]),
x2 = d3.scaleTime().domain(d3.extent(res, d => d.Time)).range([0, width]),
y = d3.scaleLinear().domain([0, SMYSKPa]).range([height, 0]),
y2 = d3.scaleLinear().domain([0, SMYSKPa]).range([height2, 0]);
//clear the content in Spectrum-fig div before drawring
//avoid multiple drawings;
var xAxis =d3.axisBottom(x).tickFormat(d3.timeFormat("%m-%d")),
xAxis2 = d3.axisBottom(x2).tickFormat(d3.timeFormat("%b")),
yAxis = d3.axisLeft(y);
var brush = d3.brushX() // Add the brush feature using the d3.brush function
.extent( [ [0,0], [width,height2] ] ) // initialise the brush area: start at 0,0 and finishes at width,height: it means I select the whole graph area
.on("brush end", brushed); // trigger the brushed function
var zoom = d3.zoom()
.scaleExtent([1, 100]) //defined the scale extend
.translateExtent([[0, 0], [width, height]])
.extent([[0, 0], [width, height]])
.on("zoom", zoomed); //at the zoom end trigger zoomed function
//line for big chart line
var line = d3.line()
.x(function(d) { return x(d.Time) })
.y(function(d) { return y(d.psi) });
//line2 for small chart line
var line2 = d3.line()
.x(function(d) { return x2(d.Time) })
.y(function(d) { return y2(d.psi) });
var svg = d3.select("#spectrum-fig")
.append("svg")
.attr("width", fullwidth)
.attr("height", fullheight);
svg.append("defs").append("clipPath")
.attr("id", "clip")
.append("rect")
.attr("width", width)
.attr("height", height);
var focus = svg.append("g")
.attr("class", "focus")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
var context = svg.append("g")
.attr("class", "context")
.attr("transform", "translate(" + margin2.left + "," + margin2.top + ")");
focus.append("g")
.attr("class", "axis axis--x")
.attr("transform", "translate (0," + height +")")
.call(xAxis);
focus.append("g")
.attr("class", "axis axis--y")
.call(yAxis);
focus.append("g")
.attr("transform", "translate (" + width + ", 0)")
.call(d3.axisRight(y).tickFormat('').tickSize(0));
focus.append("g")
.attr("transform", "translate (0, 0)")
.call(d3.axisTop(x).tickFormat('').tickSize(0));
// Add the line
focus.insert("path")
//.datum(res)
.attr("class", "line") // I add the class line to be able to modify this line later on.
.attr("fill", "none")
.attr('clip-path', 'url(#clip)')
.attr("stroke", "steelblue")
.attr("stroke-width", 1.5)
.attr("d", line(resLevelTwo));
context.insert("path")
//.datum(resSmallChart)
.attr("class", "line")
.attr("stroke", "steelblue")
.attr("stroke-width", 1.5)
.attr("fill", "none")
.attr("d", line2(resSmallChart));
context.append("g")
.attr("class", "axis axis--x")
.attr("transform", "translate(0," + height2 + ")")
.call(xAxis2);
context.append("g")
.attr("class", "brush")
.call(brush)
.call(brush.move, x.range());
svg.append("rect")
.attr("class", "zoom")
.attr('fill', 'none')
.attr('cursor', 'move')
.attr('pointer-events', 'all')
.attr("width", width)
.attr("height", height)
.attr("transform", "translate(" + margin.left + "," + margin.top + ")")
.call(zoom);
function getWindowed(arr, level){
var windowed = new Array();
var arrLength = arr.length;
var windowSize =Math.pow(16, level); //set the window size
for(let i = 0; i * windowSize < arrLength; i++ ){ //each to be the window size
let startIndex = i * windowSize;
let endIndex = (i+1) * windowSize;
endIndex = endIndex >= arrLength ? arrLength-1 : endIndex;
let localExtreme = findLocalExtreme(arr.slice(startIndex, endIndex));
if (localExtreme.Max.Time.getTime() === localExtreme.Min.Time.getTime()){ //anything include = need getTime
windowed.push(localExtreme.Max)
}else if(localExtreme.Max.Time < localExtreme.Min.Time){
windowed.push(localExtreme.Max);
windowed.push(localExtreme.Min);
}else{
windowed.push(localExtreme.Min);
windowed.push(localExtreme.Max);
}
}
let firstElement = {...arr[0]};
let lastElement = {...arr[arr.length-1]};
if(firstElement.Time.getTime() != windowed[0].Time.getTime()){ //insert to the position zero
windowed.unshift(firstElement);
}
if(lastElement.Time.getTime() != windowed[windowed.length-1].Time.getTime()){
windowed.push(lastElement);
}//insert to the end last member;
return windowed;
}
function findLocalExtreme(slicedArr){
if(slicedArr === undefined || slicedArr.length == 0){
throw 'error: no array members';
}
let slicedArrLength = slicedArr.length;
let tempMax = {...slicedArr[0]};
let tempMin = {...slicedArr[0]};
if(slicedArrLength === 1){
return {
Max: tempMax,
Min: tempMin
}
}
for (let i = 1; i < slicedArrLength; i++){
if (slicedArr[i].psi > tempMax.psi){
tempMax = {...slicedArr[i]};
}
if (slicedArr[i].psi < tempMin.psi){
tempMin = {...slicedArr[i]};
}
}
return {
Max: tempMax,
Min: tempMin
}
}
function getDataToDraw(timeRange){ //timeRange [0,1] , [startTime, endTime]
const bisect = d3.bisector(d => d.Time).left;
const startIndex = bisect(res, timeRange[0]);
const endIndex = bisect(res, timeRange[1]);
const numberInOriginal = endIndex-startIndex+1;
const windowSize =16;
const maxNumber = 8000;
let level = Math.ceil(Math.log(numberInOriginal/maxNumber ) / Math.log(windowSize));
if(level <=0 ) level =0;
console.log(endIndex, startIndex, endIndex-startIndex+1, level);
if(level === 0){
return res.slice(startIndex, endIndex);
}if(level === 1){
let start_i = bisect(resLevelOne, timeRange[0]);
let end_i =bisect(resLevelOne, timeRange[1]);
return resLevelOne.slice(start_i, end_i);
}else { //if level 2 or higher, never happen
let start_i = bisect(resLevelTwo, timeRange[0]);
let end_i =bisect(resLevelTwo, timeRange[1]);
return resLevelTwo.slice(start_i, end_i);
}
}
function brushed() {
if (d3.event.sourceEvent && d3.event.sourceEvent.type === "zoom") return; // ignore brush-by-zoom
var s = d3.event.selection || x2.range();
x.domain(s.map(x2.invert, x2));
focus.select(".line").attr("d", line(getDataToDraw(x.domain())));
focus.select(".axis--x").call(xAxis);
svg.select(".zoom").call(zoom.transform, d3.zoomIdentity
.scale(width / (s[1] - s[0]))
.translate(-s[0], 0));
}
function zoomed() {
if (d3.event.sourceEvent && d3.event.sourceEvent.type === "brush") return; // ignore zoom-by-brush
var t = d3.event.transform;
//console.log(t);
x.domain(t.rescaleX(x2).domain());
focus.select(".line").attr("d", line(getDataToDraw(t.rescaleX(x2).domain())));
focus.select(".axis--x").call(xAxis);
context.select(".brush").call(brush.move, x.range().map(t.invertX, t));
}
}
Refer to the post plotting 50 million points with d3.js.
Sluggish interaction with zoom and pan are due to too many elements in the svg. The key is to use hierarchical levels of detail, to limit the maximum elements in svg.