Search code examples
pythonkairosdb

Python script to inject bulks into KairosDB - only 1st bulk inserted, the rest ignored


I wrote the following script that allows me to set the amount of sensors and the amount of bulk inserts for them. Each bulk insert gets a different epoch time. Each sensor insert gets a different value + different tag (temp) value.

Kairos version:

root@ip-172-16-0-147:~# echo "version" | nc -w 30 localhost 4242
KairosDB 1.2.0-0.3beta.20171211170411

Scylla (Backend Storage) version:

[centos@ip-172-16-0-128 ~]$ scylla --version
2.0.2-0.20171201.07b039f

The problem: Only the 1st bulk is inserted into Kairos, the other bulks are just ignored and I can't figure out why. Any ideas as to why, or how to correct the script are appreciated.

The script:

import socket
import time
import random

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(("[Kairos_IP]", 4242))

start_time = time.time()
lowerRange = 1
upperRange = 5
sensor_amount = 10
sensors = []
for x in range(lowerRange, upperRange):
    str=""
    curr_epoch = time.time()
    #curr_epoch = int(time.time() * 1000)
    time.sleep(1)
    for y in range(1,sensor_amount):
        statement = "put SENSOR_%d %d %d temp=%d\n" %(y,curr_epoch,random.randint(1,500),random.randint(1,500))
        sensors.append(statement)
        str+=(sensors[y-1])
        print statement
        s.send(str)
    print curr_epoch

print("--- %s seconds ---" % (time.time() - start_time))

Example of output:

put SENSOR_1 1515672684 240 temp=500
put SENSOR_2 1515672684 279 temp=403
put SENSOR_3 1515672684 380 temp=376
put SENSOR_4 1515672684 59 temp=38
put SENSOR_5 1515672684 405 temp=46
put SENSOR_6 1515672684 61 temp=255
put SENSOR_7 1515672684 19 temp=21
put SENSOR_8 1515672684 246 temp=80
put SENSOR_9 1515672684 289 temp=233
1515672684.72
put SENSOR_1 1515672685 58 temp=12
put SENSOR_2 1515672685 245 temp=199
put SENSOR_3 1515672685 492 temp=220
put SENSOR_4 1515672685 185 temp=432
put SENSOR_5 1515672685 235 temp=416
put SENSOR_6 1515672685 218 temp=297
put SENSOR_7 1515672685 378 temp=3
put SENSOR_8 1515672685 317 temp=397
put SENSOR_9 1515672685 103 temp=229
1515672685.72
put SENSOR_1 1515672686 482 temp=395
put SENSOR_2 1515672686 417 temp=7
put SENSOR_3 1515672686 285 temp=111
put SENSOR_4 1515672686 67 temp=322
put SENSOR_5 1515672686 402 temp=3
put SENSOR_6 1515672686 120 temp=410
put SENSOR_7 1515672686 232 temp=50
put SENSOR_8 1515672686 90 temp=263
put SENSOR_9 1515672686 315 temp=48
1515672686.72
put SENSOR_1 1515672687 181 temp=453
put SENSOR_2 1515672687 212 temp=414
put SENSOR_3 1515672687 116 temp=138
put SENSOR_4 1515672687 106 temp=118
put SENSOR_5 1515672687 92 temp=348
put SENSOR_6 1515672687 178 temp=361
put SENSOR_7 1515672687 148 temp=198
put SENSOR_8 1515672687 334 temp=79
put SENSOR_9 1515672687 191 temp=192
1515672687.72

Solution

  • Found the problem - here is the correct version of the script:

    import socket
    import time
    import random
    
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.connect(("[Kairos_IP]", 4242))
    
    start_time = time.time()
    
    sensor_amount = 501
    #sensors = []
    #curr_epoch = int(round(time.time() * 1000))
    
    for num_repeat in range(1,501):
            str=""
            time.sleep(2)
            curr_epoch = int(round(time.time() * 1000))
            print "Current time " + time.strftime("%X")
            print "======================"
            for y in range(1,sensor_amount):
                    statement = "putm SENSOR_%d %d %d temp=%d\n" %(y,curr_epoch,random.randint(1,500),random.randint(1,500))
                    print statement
                    s.send(statement)
    
    print("--- %s seconds ---" % (time.time() - start_time))