I am new both in Python and ibapi, I have ask Interactive Broker itself, they told me only :
The self.done is originally defined in the parent class EClient that is used to indicate the current API connection is "done". You should not use it as an indicator to stop a loop that is build by yourself. You can keep one API client connection live until you have finished downloaded all the historical data you need. If you need to slow down the pace in requesting data, you can use other thread control in python, such as the sleep() function, to add more wait time in your loop so it does not send historical data requests for all contracts at once.
So I think that the error came from
self.done = True
This is my code
from ibapi import wrapper
from ibapi.client import EClient
from ibapi.wrapper import EWrapper
from ibapi.contract import Contract as IBcontract
from threading import Thread
import queue
import datetime
from ibapi.utils import iswrapper #just for decorator
from ibapi.common import *
from ibapi.contract import *
from ibapi.ticktype import *
class TestApp(wrapper.EWrapper, EClient):
def __init__(self):
wrapper.EWrapper.__init__(self)
EClient.__init__(self, wrapper=self)
@iswrapper
def historicalData(self, reqId:int, bar: BarData):
print("HistoricalData. ", reqId, " Date:", bar.date, "Open:", bar.open,
"High:", bar.high, "Low:", bar.low, "Close:", bar.close, "Volume:", bar.volume,
"Count:", bar.barCount, "WAP:", bar.average)
if all_data.iloc[reqId,7] == 0:
all_data.iloc[reqId,7] = bar.close
self.done = True
@iswrapper
def historicalDataEnd(self, reqId: int, start: str, end: str):
super().historicalDataEnd(reqId, start, end)
print("HistoricalDataEnd ", reqId, "from", start, "to", end)
@iswrapper
def historicalDataUpdate(self, reqId: int, bar: BarData):
print("HistoricalDataUpdate. ", reqId, " Date:", bar.date, "Open:", bar.open,
"High:", bar.high, "Low:", bar.low, "Close:", bar.close, "Volume:", bar.volume,
"Count:", bar.barCount, "WAP:", bar.average)
def main():
t = time()
max_amount_per_Iter = 70 #max number per iter to save cost
max_Iter = ceil(len(all_data)/max_amount_per_Iter)
for i in range (0,max_Iter):
print('====================round : ',i+1,'===============================')
app = TestApp()
app.connect("127.0.0.1", 7496, clientId=i)
print("serverVersion:%s connectionTime:%s" % (app.serverVersion(),app.twsConnectionTime()))
for j in range (0,min(max_amount_per_Iter,len(all_data)-i*max_amount_per_Iter)):
print(j+i*70)
app.i = j+i*max_amount_per_Iter
contract = Contract()
contract.symbol = all_data.iloc[app.i,0]
contract.secType = all_data.iloc[app.i,1]
contract.currency = all_data.iloc[app.i,3]
contract.exchange = all_data.iloc[app.i,2]
app.reqHistoricalData(app.i, contract, "","1 W", "1 day", "Adjusted_Last", 1, 1, False, [])
app.run()
sleep(1)
app.disconnect()
sleep(0.02)
print('=========End round : ',i+1,'with time :',time() - t,'==============')
if __name__ == "__main__":
main()
I hope to see a suggestion how to pass to the next for iteration without using self.done or disconnect my TWS. If I don't include self.done the program will run on an infinite loop at first i=0,j=0 iteration, nobody tells it to quit.
So if you don't know the direct answer that can solve mine, you can suggest:
Is there other flow control that can use in this case?
Or the cancelHistoricalData command, if it can use for this case?
Or is there any way to build Class in Class and execute self.done = True only in a subclass with out disconnect my ID in Class?
PS. I have the same problem with requestMarketData
Update V2 This is some result that I try to print for
====================round : 1 ===============================
step2
serverVersion:124 connectionTime:b'20170821 22:34:09 ICT'
0
ERROR:root:ERROR -1 2104 Market data farm connection is OK:hfarm
ERROR:root:ERROR -1 2104 Market data farm connection is OK:jfarm
ERROR:root:ERROR -1 2104 Market data farm connection is OK:usfuture
ERROR:root:ERROR -1 2104 Market data farm connection is OK:eufarm
ERROR:root:ERROR -1 2104 Market data farm connection is OK:cashfarm
ERROR:root:ERROR -1 2104 Market data farm connection is OK:usfarm.us
ERROR:root:ERROR -1 2104 Market data farm connection is OK:usfarm
ERROR:root:ERROR -1 2106 HMDS data farm connection is OK:ilhmds
ERROR:root:ERROR -1 2106 HMDS data farm connection is OK:euhmds
ERROR:root:ERROR -1 2106 HMDS data farm connection is OK:fundfarm
ERROR:root:ERROR -1 2106 HMDS data farm connection is OK:ushmds
HistoricalData. 0 Date: 20170815 Open: 67.2 High: 68.24 Low: 66.99 Close: 68.02 Volume: 13268 Count: 9453 WAP: 67.8105
HistoricalData. 0 Date: 20170816 Open: 68.28 High: 68.91 Low: 67.45 Close: 68.01 Volume: 11950 Count: 9723 WAP: 68.457
HistoricalData. 0 Date: 20170817 Open: 67.8 High: 68.53 Low: 66.83 Close: 66.89 Volume: 11407 Count: 9432 WAP: 67.504
HistoricalData. 0 Date: 20170818 Open: 66.91 High: 67.25 Low: 66.57 Close: 66.78 Volume: 12091 Count: 9637 WAP: 66.8445
HistoricalData. 0 Date: 20170821 Open: 66.9 High: 66.96 Low: 66.14 Close: 66.28 Volume: 3317 Count: 2541 WAP: 66.3425
HistoricalDataEnd 0 from 20170814 22:34:14 to 20170821 22:34:14
ERROR:root:ERROR 1 504 Not connected
ERROR:root:ERROR 2 504 Not connected
ERROR:root:ERROR 3 504 Not connected
ERROR:root:ERROR 4 504 Not connected
ERROR:root:ERROR 5 504 Not connected
ERROR:root:ERROR 6 504 Not connected
ERROR:root:ERROR 7 504 Not connected
ERROR:root:ERROR 8 504 Not connected
ERROR:root:ERROR 9 504 Not connected
ERROR:root:ERROR 10 504 Not connected
ERROR:root:ERROR 11 504 Not connected
ERROR:root:ERROR 12 504 Not connected
ERROR:root:ERROR 13 504 Not connected
ERROR:root:ERROR 14 504 Not connected
ERROR:root:ERROR 15 504 Not connected
ERROR:root:ERROR 16 504 Not connected
ERROR:root:ERROR 17 504 Not connected
ERROR:root:ERROR 18 504 Not connected
ERROR:root:ERROR 19 504 Not connected
ERROR:root:ERROR 20 504 Not connected
ERROR:root:ERROR 21 504 Not connected
ERROR:root:ERROR 22 504 Not connected
ERROR:root:ERROR 23 504 Not connected
ERROR:root:ERROR 24 504 Not connected
ERROR:root:ERROR 25 504 Not connected
ERROR:root:ERROR 26 504 Not connected
ERROR:root:ERROR 27 504 Not connected
ERROR:root:ERROR 28 504 Not connected
ERROR:root:ERROR 29 504 Not connected
ERROR:root:ERROR 30 504 Not connected
ERROR:root:ERROR 31 504 Not connected
ERROR:root:ERROR 32 504 Not connected
ERROR:root:ERROR 33 504 Not connected
ERROR:root:ERROR 34 504 Not connected
ERROR:root:ERROR 35 504 Not connected
ERROR:root:ERROR 36 504 Not connected
ERROR:root:ERROR 37 504 Not connected
ERROR:root:ERROR 38 504 Not connected
ERROR:root:ERROR 39 504 Not connected
ERROR:root:ERROR 40 504 Not connected
ERROR:root:ERROR 41 504 Not connected
ERROR:root:ERROR 42 504 Not connected
ERROR:root:ERROR 43 504 Not connected
ERROR:root:ERROR 44 504 Not connected
ERROR:root:ERROR 45 504 Not connected
ERROR:root:ERROR 46 504 Not connected
ERROR:root:ERROR 47 504 Not connected
ERROR:root:ERROR 48 504 Not connected
ERROR:root:ERROR 49 504 Not connected
ERROR:root:ERROR 50 504 Not connected
ERROR:root:ERROR 51 504 Not connected
ERROR:root:ERROR 52 504 Not connected
ERROR:root:ERROR 53 504 Not connected
ERROR:root:ERROR 54 504 Not connected
ERROR:root:ERROR 55 504 Not connected
ERROR:root:ERROR 56 504 Not connected
ERROR:root:ERROR 57 504 Not connected
ERROR:root:ERROR 58 504 Not connected
ERROR:root:ERROR 59 504 Not connected
ERROR:root:ERROR 60 504 Not connected
ERROR:root:ERROR 61 504 Not connected
ERROR:root:ERROR 62 504 Not connected
ERROR:root:ERROR 63 504 Not connected
ERROR:root:ERROR 64 504 Not connected
ERROR:root:ERROR 65 504 Not connected
ERROR:root:ERROR 66 504 Not connected
ERROR:root:ERROR 67 504 Not connected
ERROR:root:ERROR 68 504 Not connected
ERROR:root:ERROR 69 504 Not connected
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
=========End round : 1 with time : 7.807971477508545 ==============
====================round : 2 ===============================
step2
ERROR:root:ERROR -1 2104 Market data farm connection is OK:hfarm
ERROR:root:ERROR -1 2104 Market data farm connection is OK:jfarm
ERROR:root:ERROR -1 2104 Market data farm connection is OK:usfuture
ERROR:root:ERROR -1 2104 Market data farm connection is OK:eufarm
ERROR:root:ERROR -1 2104 Market data farm connection is OK:cashfarm
ERROR:root:ERROR -1 2104 Market data farm connection is OK:usfarm.us
ERROR:root:ERROR -1 2104 Market data farm connection is OK:usfarm
ERROR:root:ERROR -1 2106 HMDS data farm connection is OK:ilhmds
ERROR:root:ERROR -1 2106 HMDS data farm connection is OK:euhmds
ERROR:root:ERROR -1 2106 HMDS data farm connection is OK:fundfarm
ERROR:root:ERROR -1 2106 HMDS data farm connection is OK:ushmds
serverVersion:124 connectionTime:b'20170821 22:34:17 ICT'
70
HistoricalData. 70 Date: 20170815 Open: 117.23 High: 117.62 Low: 116.58 Close: 117.43 Volume: 7232 Count: 6205 WAP: 117.1295
HistoricalData. 70 Date: 20170816 Open: 117.49 High: 119.59 Low: 117.03 Close: 119.25 Volume: 16468 Count: 11498 WAP: 118.8035
HistoricalData. 70 Date: 20170817 Open: 119.19 High: 119.48 Low: 116.46 Close: 116.47 Volume: 12285 Count: 10072 WAP: 117.4645
HistoricalData. 70 Date: 20170818 Open: 116.0 High: 117.84 Low: 115.46 Close: 116.88 Volume: 14917 Count: 10824 WAP: 116.9795
HistoricalData. 70 Date: 20170821 Open: 116.85 High: 117.4 Low: 116.15 Close: 116.77 Volume: 3471 Count: 2599 WAP: 116.5535
HistoricalDataEnd 70 from 20170814 22:34:19 to 20170821 22:34:19
ERROR:root:ERROR 71 504 Not connected
ERROR:root:ERROR 72 504 Not connected
I'd suggest a little redesign here.
In your example, you disconnect from the API by using self.done=True right after you get the first data. Since the requested data arrives in very rapid events, you get all data for the request before you do so. The result seen in your output: you got and processed all data of the first request, then you found yourself disconnected. This repeats by for i...
Remove the 'self.done=True' line. This does anything but good, you don't want that.
Restructure your code, so it waits for the historicalDataEnd event before sends the next request to the API. The API can process parallel requests, but the server does not tolerate very frequent historical data requests. If you send requests too fast, you'll get pacing violation error. Some sleep() between requests might be necessary too.
I assume app.run() waits for the disconnection, that's why you needed the 'done' flag manipulation. Remove that line too. If this call is really necessary to catch the messages (I don't know your TestApp class and its base), then make sure you call disconnect() after all your requested data arrived and processed, practically in historicalDataEnd() . I this case, a reconnection needed before the next loop, which is now missing after app.run() call within for j...
Here are my suggested modifications:
from ibapi import wrapper
from ibapi.client import EClient
from ibapi.wrapper import EWrapper
from ibapi.contract import Contract as IBcontract
from threading import Thread
import queue
import datetime
from ibapi.utils import iswrapper #just for decorator
from ibapi.common import *
from ibapi.contract import *
from ibapi.ticktype import *
class TestApp(wrapper.EWrapper, EClient):
def __init__(self):
wrapper.EWrapper.__init__(self)
EClient.__init__(self, wrapper=self)
@iswrapper
def historicalData(self, reqId:int, bar: BarData):
print("HistoricalData. ", reqId, " Date:", bar.date, "Open:", bar.open,
"High:", bar.high, "Low:", bar.low, "Close:", bar.close, "Volume:", bar.volume,
"Count:", bar.barCount, "WAP:", bar.average)
if all_data.iloc[reqId,7] == 0:
all_data.iloc[reqId,7] = bar.close
# the line below is not necessary
#self.done = True
@iswrapper
def historicalDataEnd(self, reqId: int, start: str, end: str):
super().historicalDataEnd(reqId, start, end)
print("HistoricalDataEnd ", reqId, "from", start, "to", end)
print('=========End round : ',app.i+1,'with time :',time() - app.t,'==============')
app.i++
if app.i==len(all_data):
app.disconnect() # Disconnect here: get out of app.run() when all data processed
else:
# add some sleep here if necessary
reqNext()
@iswrapper
def historicalDataUpdate(self, reqId: int, bar: BarData):
print("HistoricalDataUpdate. ", reqId, " Date:", bar.date, "Open:", bar.open,
"High:", bar.high, "Low:", bar.low, "Close:", bar.close, "Volume:", bar.volume,
"Count:", bar.barCount, "WAP:", bar.average)
def reqNext():
print('====================round : ',app.i+1,'===============================')
contract = Contract()
contract.symbol = all_data.iloc[app.i,0]
contract.secType = all_data.iloc[app.i,1]
contract.currency = all_data.iloc[app.i,3]
contract.exchange = all_data.iloc[app.i,2]
app.reqHistoricalData(app.i, contract, "","1 W", "1 day", "Adjusted_Last", 1, 1, False, [])
def main():
app = TestApp()
app.connect("127.0.0.1", 7496, clientId=1234)
print("serverVersion:%s connectionTime:%s" % (app.serverVersion(),app.twsConnectionTime()))
app.i = 0
app.t = time()
reqNext()
app.run()
# when we arrive here, app is disconnected already
if __name__ == "__main__":
main()