Search code examples
jsonchartsvisualizationvega-litevega

sample or density transform based on zoom domain

I have a large timeseries dataset that when zoomed-out brings the interactive plot to a crawl. Has anyone experimented with dynamically reducing the dataset size based on the zoom level? Such as the sampling transform where the sample number applies to the visible data not the entire dataset, but then disabled beyond a certain domain size. Is such a thing possible?

Here is a mock-up:

selection details below legend

This is how I generated the data:

dt2 = datetime.datetime.now()
ydt = datetime.timedelta(days=365)
dt1 = dt2 - ydt
dt1 = dt1.timestamp()
dt2 = dt2.timestamp()
times = [random.uniform(dt1, dt2) for i in range(10**5)]
times.sort()
values = []
for t in times:
    d = {}
    t = datetime.datetime.fromtimestamp(t)
    t = t.isoformat(" ", timespec="seconds")
    v = random.uniform(0, 1000)
    c = random.randint(0,1)
    d["time"] = t
    d["data"] = v
    d["category"] = c
    values.append(d)

with open("./test.json", "wt") as f:
    json.dump(values[:10**4], f)

And here is what I've been able to achieve in the Vega Editor (I had to trim the data to fit the link).

Solution

  • Signals are not officially supported in VL but you can hack them in. Here, the span of the domain is calculated and depending on the value, a sample is taken. You can obviously expand the ternary statement with more conditions or come up with an expression for your logic.

    {
  "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
  "data": {
    "values": [
      {"time": "2023-08-31 15:12:40", "data": 265.1037232391961, "category": 1},
      {"time": "2023-08-31 15:15:26", "data": 989.2391954577464, "category": 1},
      {"time": "2023-08-31 15:15:29", "data": 426.3748533977788, "category": 0},
      {"time": "2023-08-31 15:18:11", "data": 563.7725296786067, "category": 1},
      {"time": "2023-08-31 15:21:25", "data": 322.4493083566362, "category": 0},
      {"time": "2023-08-31 15:24:54", "data": 822.6771646740021, "category": 1},
      {
        "time": "2023-08-31 15:28:10",
        "data": 294.37404484299054,
        "category": 1
      },
      {"time": "2023-08-31 15:28:22", "data": 838.7462086185608, "category": 0},
      {"time": "2023-08-31 15:35:58", "data": 961.3893188770259, "category": 0},
      {
        "time": "2023-08-31 15:36:07",
        "data": 241.45631836625802,
        "category": 0
      },
      {
        "time": "2023-08-31 15:49:33",
        "data": 191.96326506124362,
        "category": 0
      },
      {
        "time": "2023-08-31 15:51:15",
        "data": 450.50733664623965,
        "category": 1
      },
      {"time": "2023-08-31 15:56:35", "data": 390.5921971631632, "category": 1},
      {"time": "2023-08-31 15:57:52", "data": 829.8364876130439, "category": 1},
      {"time": "2023-08-31 16:01:06", "data": 996.0349700996576, "category": 0},
      {"time": "2023-08-31 16:02:36", "data": 78.24722444300802, "category": 0},
      {"time": "2023-08-31 16:14:05", "data": 942.3350040849994, "category": 0},
      {"time": "2023-08-31 16:14:56", "data": 860.58714895142, "category": 1},
      {"time": "2023-08-31 16:15:08", "data": 515.199102407516, "category": 1},
      {
        "time": "2023-08-31 16:23:20",
        "data": 166.05721829849873,
        "category": 1
      },
      {
        "time": "2023-08-31 16:30:05",
        "data": 439.73137493646266,
        "category": 0
      },
      {"time": "2023-08-31 16:32:31", "data": 869.245076742056, "category": 0},
      {
        "time": "2023-08-31 16:35:48",
        "data": 480.50968063008304,
        "category": 1
      },
      {"time": "2023-08-31 16:37:50", "data": 476.877035209344, "category": 1},
      {"time": "2023-08-31 16:39:36", "data": 733.3017448826324, "category": 0},
      {"time": "2023-08-31 16:44:17", "data": 636.686519092496, "category": 1},
      {"time": "2023-08-31 16:45:08", "data": 694.5261775005811, "category": 0},
      {"time": "2023-08-31 16:51:36", "data": 695.7401884245502, "category": 0},
      {"time": "2023-08-31 16:55:29", "data": 570.0935946720598, "category": 1},
      {
        "time": "2023-08-31 16:57:05",
        "data": 277.22052647262717,
        "category": 0
      },
      {
        "time": "2023-08-31 16:58:27",
        "data": 480.36926264607274,
        "category": 1
      },
      {"time": "2023-08-31 17:02:34", "data": 893.3698570026319, "category": 1},
      {
        "time": "2023-08-31 17:05:32",
        "data": 236.71895124154685,
        "category": 1
      },
      {"time": "2023-08-31 17:08:46", "data": 573.0841835923452, "category": 0},
      {"time": "2023-08-31 17:14:37", "data": 191.7254918774728, "category": 0},
      {"time": "2023-08-31 17:16:43", "data": 94.93763899240804, "category": 1},
      {"time": "2023-08-31 17:24:40", "data": 936.4038465823089, "category": 0},
      {
        "time": "2023-08-31 17:31:09",
        "data": 390.84825100994567,
        "category": 1
      },
      {"time": "2023-08-31 17:35:35", "data": 14.48187309843274, "category": 0},
      {
        "time": "2023-08-31 17:35:47",
        "data": 443.05398617944593,
        "category": 1
      },
      {"time": "2023-08-31 17:40:44", "data": 30.0828399028229, "category": 0},
      {"time": "2023-08-31 17:48:33", "data": 768.0549896500464, "category": 1},
      {"time": "2023-08-31 17:53:29", "data": 71.57068127924227, "category": 0},
      {"time": "2023-08-31 18:04:56", "data": 594.7138236213322, "category": 0},
      {"time": "2023-08-31 18:06:44", "data": 29.21370270526036, "category": 0},
      {"time": "2023-08-31 18:28:22", "data": 852.7093808483378, "category": 1},
      {"time": "2023-08-31 18:30:01", "data": 576.9728506525139, "category": 1},
      {"time": "2023-08-31 18:31:41", "data": 968.1882202042807, "category": 1},
      {"time": "2023-08-31 18:31:51", "data": 185.6873327854428, "category": 1},
      {"time": "2023-08-31 18:33:31", "data": 258.211113709635, "category": 0},
      {"time": "2023-08-31 18:36:36", "data": 641.264570256715, "category": 1},
      {"time": "2023-08-31 18:39:52", "data": 717.6143367808544, "category": 1},
      {"time": "2023-08-31 18:39:52", "data": 191.4611806426172, "category": 1},
      {"time": "2023-08-31 18:41:38", "data": 136.9116350629923, "category": 0},
      {"time": "2023-08-31 18:57:48", "data": 62.11343548023751, "category": 1},
      {"time": "2023-08-31 18:58:26", "data": 529.5089127094398, "category": 0},
      {
        "time": "2023-08-31 19:07:54",
        "data": 153.13269404700824,
        "category": 1
      },
      {"time": "2023-08-31 19:09:17", "data": 705.4049459845114, "category": 0},
      {
        "time": "2023-08-31 19:11:07",
        "data": 300.90132125121005,
        "category": 1
      },
      {"time": "2023-08-31 19:20:25", "data": 946.4725291504993, "category": 1},
      {
        "time": "2023-08-31 19:23:48",
        "data": 319.04133613813724,
        "category": 1
      },
      {"time": "2023-08-31 19:24:25", "data": 464.2923297748929, "category": 1},
      {"time": "2023-08-31 19:28:02", "data": 836.436678063193, "category": 1},
      {"time": "2023-08-31 19:28:08", "data": 5.992853044164859, "category": 1},
      {"time": "2023-08-31 19:40:01", "data": 873.6847072580948, "category": 1},
      {"time": "2023-08-31 19:43:41", "data": 431.0286183407737, "category": 1},
      {
        "time": "2023-08-31 19:51:22",
        "data": 396.43260404732825,
        "category": 0
      },
      {"time": "2023-08-31 19:54:08", "data": 575.9715221353141, "category": 0},
      {
        "time": "2023-08-31 19:55:53",
        "data": 44.016217670442614,
        "category": 0
      },
      {"time": "2023-08-31 19:58:14", "data": 988.9639046666363, "category": 1},
      {"time": "2023-08-31 20:05:53", "data": 742.2798696276691, "category": 0},
      {"time": "2023-08-31 20:07:13", "data": 982.7119961613008, "category": 0},
      {"time": "2023-08-31 20:15:20", "data": 976.3381077100345, "category": 1},
      {"time": "2023-08-31 20:20:15", "data": 498.5276910780252, "category": 0},
      {"time": "2023-08-31 20:22:29", "data": 301.4863894174468, "category": 1},
      {
        "time": "2023-08-31 20:31:03",
        "data": 232.56452406666895,
        "category": 1
      },
      {"time": "2023-08-31 20:33:52", "data": 694.171014904713, "category": 1},
      {
        "time": "2023-08-31 20:35:45",
        "data": 102.79567934930212,
        "category": 1
      },
      {
        "time": "2023-08-31 20:47:32",
        "data": 431.64822699883376,
        "category": 1
      },
      {"time": "2023-08-31 20:55:19", "data": 683.217576875891, "category": 0},
      {"time": "2023-08-31 20:55:36", "data": 879.5945045918183, "category": 1},
      {"time": "2023-08-31 21:04:28", "data": 164.6834561802648, "category": 1},
      {
        "time": "2023-08-31 21:06:04",
        "data": 22.588620229922583,
        "category": 1
      },
      {"time": "2023-08-31 21:07:10", "data": 757.0796861192514, "category": 1},
      {"time": "2023-08-31 21:23:43", "data": 848.456892794343, "category": 1},
      {
        "time": "2023-08-31 21:34:38",
        "data": 447.89147371830785,
        "category": 1
      },
      {"time": "2023-08-31 21:45:30", "data": 862.3116375036777, "category": 1},
      {"time": "2023-08-31 21:47:00", "data": 967.0312319533795, "category": 0},
      {"time": "2023-08-31 21:47:56", "data": 966.4938018703745, "category": 1},
      {"time": "2023-08-31 21:49:45", "data": 890.2567189914545, "category": 0},
      {
        "time": "2023-08-31 21:55:40",
        "data": 362.80312104639677,
        "category": 1
      },
      {"time": "2023-08-31 21:58:55", "data": 834.7469369912607, "category": 1},
      {"time": "2023-08-31 22:01:02", "data": 584.1447613550432, "category": 1},
      {"time": "2023-08-31 22:01:06", "data": 82.66592460479994, "category": 1},
      {
        "time": "2023-08-31 22:02:00",
        "data": 332.67959271479384,
        "category": 0
      },
      {
        "time": "2023-08-31 22:02:32",
        "data": 316.51081491367347,
        "category": 0
      },
      {
        "time": "2023-08-31 22:08:31",
        "data": 336.10098602094985,
        "category": 1
      },
      {"time": "2023-08-31 22:18:52", "data": 873.7313013506864, "category": 0},
      {
        "time": "2023-08-31 22:19:24",
        "data": 312.42947148514776,
        "category": 1
      },
      {"time": "2023-08-31 22:28:48", "data": 582.8096654568776, "category": 1}
    ]
  },
  "params": [{"name": "test", "expr": "span(grid_time)/100000"}],
  "transform": [
    {"filter": "datum.data > 0"},
    {"sample": {"signal": "test<130?100:10"}}
  ],
  "title": "Too Much Data",
  "config": {"font": "monospace"},
  "width": "container",
  "layer": [
    {
      "params": [
        {
          "name": "grid",
          "bind": "scales",
          "select": {
            "type": "interval",
            "encodings": ["x"],
            "on": "[mousedown[!event.shiftKey], mouseup] > mousemove",
            "translate": "[mousedown[!event.shiftKey], mouseup] > mousemove!"
          }
        },
        {
          "name": "brush",
          "select": {
            "type": "interval",
            "encodings": ["x"],
            "on": "[mousedown[event.shiftKey], mouseup] > mousemove",
            "translate": "[mousedown[event.shiftKey], mouseup] > mousemove!"
          }
        }
      ],
      "mark": "point",
      "encoding": {
        "x": {"field": "time", "type": "temporal"},
        "y": {"field": "data", "type": "quantitative"},
        "color": {"field": "category", "type": "nominal"}
      }
    },
    {
      "transform": [
        {"filter": {"param": "brush"}},
        {"aggregate": [{"op": "count", "as": "count"}]}
      ],
      "mark": "text",
      "encoding": {"text": {"field": "count", "type": "quantitative"}}
    }
  ]
}