I have a python script I use to grab images from an ip camera through my home network and add date time information. In a 12 hour period it grabs about 200,000 pictures. But when using zoneminder (camera monitoring software) the camera manages 250,000 in a 7 hour period.
I was wondering if anyone could help me improve my script efficiency I have tried using the threading module to create 2 threads but it has not helped i am not sure if I have implemented it wrong or not. Below is code I am currently using:
#!/usr/bin/env python
# My First python script to grab images from an ip camera
import requests
import time
import urllib2
import sys
import os
import PIL
from PIL import ImageFont
from PIL import Image
from PIL import ImageDraw
import datetime
from datetime import datetime
import threading
timecount = 43200
lock = threading.Lock()
wdir = "/workdir/"
y = len([f for f in os.listdir(wdir)
if f.startswith('Cam1') and os.path.isfile(os.path.join(wdir, f))])
def looper(timeCount):
global y
start = time.time()
keepLooping = True
while keepLooping:
with lock:
y += 1
now = datetime.now()
dte = str(now.day) + ":" + str(now.month) + ":" + str(now.year)
dte1 = str(now.hour) + ":" + str(now.minute) + ":" + str(now.second) + "." + str(now.microsecond)
cname = "Cam1:"
dnow = """Date: %s """ % (dte)
dnow1 = """Time: %s""" % (dte1)
buffer = urllib2.urlopen('http://(ip address)/snapshot.cgi?user=uname&pwd=password').read()
img = str(wdir) + "Cam1-" + str('%010d' % y) + ".jpg"
f = open(img, 'wb')
f.write(buffer)
f.close()
if time.time()-start > timeCount:
keepLooping = False
font = ImageFont.truetype("/usr/share/fonts/truetype/ttf-dejavu/DejaVuSans.ttf",10)
img=Image.open(img)
draw = ImageDraw.Draw(img)
draw.text((0, 0),cname,fill="white",font=font)
draw.text((0, 10),dnow,fill="white",font=font)
draw.text((0, 20),dnow1,fill="white",font=font)
draw = ImageDraw.Draw(img)
draw = ImageDraw.Draw(img)
img.save(str(wdir) + "Cam1-" + str('%010d' % y) + ".jpg")
for i in range(2):
thread = threading.Thread(target=looper,args=(timecount,))
thread.start()
thread.join()
how could i improve this script or how do i open a stream from the camera then grab images from the stream? would that even increase the efficiency / capture rate?
Edit:
Thanks to kobejohn's help i have come up with the following implementation. running for a 12 hour period it has gotten over 420,000 pictures from 2 seperate cameras (at the same tme) each running on their own thread at the same time compared to about 200,000 from my origional implementation above. The following code will run 2 camera's in parallel (or close enough to it) and add text to them:
import base64
from datetime import datetime
import httplib
import io
import os
import time
from PIL import ImageFont
from PIL import Image
from PIL import ImageDraw
import multiprocessing
wdir = "/workdir/"
stream_urlA = '192.168.3.21'
stream_urlB = '192.168.3.23'
usernameA = ''
usernameB = ''
password = ''
y = sum(1 for f in os.listdir(wdir) if f.startswith('CamA') and os.path.isfile(os.path.join(wdir, f)))
x = sum(1 for f in os.listdir(wdir) if f.startswith('CamB') and os.path.isfile(os.path.join(wdir, f)))
def main():
time_count = 43200
# time_count = 1
procs = list()
for i in range(1):
p = multiprocessing.Process(target=CameraA, args=(time_count, y,))
q = multiprocessing.Process(target=CameraB, args=(time_count, x,))
procs.append(p)
procs.append(q)
p.start()
q.start()
for p in procs:
p.join()
def CameraA(time_count, y):
y = y
h = httplib.HTTP(stream_urlA)
h.putrequest('GET', '/videostream.cgi')
h.putheader('Authorization', 'Basic %s' % base64.encodestring('%s:%s' % (usernameA, password))[:-1])
h.endheaders()
errcode, errmsg, headers = h.getreply()
stream_file = h.getfile()
start = time.time()
end = start + time_count
while time.time() <= end:
y += 1
now = datetime.now()
dte = str(now.day) + "-" + str(now.month) + "-" + str(now.year)
dte1 = str(now.hour) + ":" + str(now.minute) + ":" + str(now.second) + "." + str(now.microsecond)
cname = "Cam#: CamA"
dnow = """Date: %s """ % dte
dnow1 = """Time: %s""" % dte1
# your camera may have a different streaming format
# but I think you can figure it out from the debug style below
source_name = stream_file.readline() # '--ipcamera'
content_type = stream_file.readline() # 'Content-Type: image/jpeg'
content_length = stream_file.readline() # 'Content-Length: 19565'
#print 'confirm/adjust content (source?): ' + source_name
#print 'confirm/adjust content (type?): ' + content_type
#print 'confirm/adjust content (length?): ' + content_length
# find the beginning of the jpeg data BEFORE pulling the jpeg framesize
# there must be a more efficient way, but hopefully this is not too bad
b1 = b2 = b''
while True:
b1 = stream_file.read(1)
while b1 != chr(0xff):
b1 = stream_file.read(1)
b2 = stream_file.read(1)
if b2 == chr(0xd8):
break
# pull the jpeg data
framesize = int(content_length[16:])
jpeg_stripped = b''.join((b1, b2, stream_file.read(framesize - 2)))
# throw away the remaining stream data. Sorry I have no idea what it is
junk_for_now = stream_file.readline()
# convert directly to an Image instead of saving / reopening
# thanks to SO: http://stackoverflow.com/a/12020860/377366
image_as_file = io.BytesIO(jpeg_stripped)
image_as_pil = Image.open(image_as_file)
draw = ImageDraw.Draw(image_as_pil)
draw.text((0, 0), cname, fill="white")
draw.text((0, 10), dnow, fill="white")
draw.text((0, 20), dnow1, fill="white")
img_name = "CamA-" + str('%010d' % y) + ".jpg"
img_path = os.path.join(wdir, img_name)
image_as_pil.save(img_path)
def CameraB(time_count, x):
x = x
h = httplib.HTTP(stream_urlB)
h.putrequest('GET', '/videostream.cgi')
h.putheader('Authorization', 'Basic %s' % base64.encodestring('%s:%s' % (usernameB, password))[:-1])
h.endheaders()
errcode, errmsg, headers = h.getreply()
stream_file = h.getfile()
start = time.time()
end = start + time_count
while time.time() <= end:
x += 1
now = datetime.now()
dte = str(now.day) + "-" + str(now.month) + "-" + str(now.year)
dte1 = str(now.hour) + ":" + str(now.minute) + ":" + str(now.second) + "." + str(now.microsecond)
cname = "Cam#: CamB"
dnow = """Date: %s """ % dte
dnow1 = """Time: %s""" % dte1
# your camera may have a different streaming format
# but I think you can figure it out from the debug style below
source_name = stream_file.readline() # '--ipcamera'
content_type = stream_file.readline() # 'Content-Type: image/jpeg'
content_length = stream_file.readline() # 'Content-Length: 19565'
#print 'confirm/adjust content (source?): ' + source_name
#print 'confirm/adjust content (type?): ' + content_type
#print 'confirm/adjust content (length?): ' + content_length
# find the beginning of the jpeg data BEFORE pulling the jpeg framesize
# there must be a more efficient way, but hopefully this is not too bad
b1 = b2 = b''
while True:
b1 = stream_file.read(1)
while b1 != chr(0xff):
b1 = stream_file.read(1)
b2 = stream_file.read(1)
if b2 == chr(0xd8):
break
# pull the jpeg data
framesize = int(content_length[16:])
jpeg_stripped = b''.join((b1, b2, stream_file.read(framesize - 2)))
# throw away the remaining stream data. Sorry I have no idea what it is
junk_for_now = stream_file.readline()
# convert directly to an Image instead of saving / reopening
# thanks to SO: http://stackoverflow.com/a/12020860/377366
image_as_file = io.BytesIO(jpeg_stripped)
image_as_pil = Image.open(image_as_file)
draw = ImageDraw.Draw(image_as_pil)
draw.text((0, 0), cname, fill="white")
draw.text((0, 10), dnow, fill="white")
draw.text((0, 20), dnow1, fill="white")
img_name = "CamB-" + str('%010d' % x) + ".jpg"
img_path = os.path.join(wdir, img_name)
image_as_pil.save(img_path)
if __name__ == '__main__':
main()
EDIT (26/05/2014):
I have spent the better part of 2 months trying to update this script / program to work with python 3 but have been completely unable to get it to do anything. would anyone be able to point me in the right direction?
I have tried the 2to3 script but it just changed a couple of entries and I still was unable to get it to function at all.
*edit I previously blamed GIL for the behavior which was silly. This is an I/O bound process, not a CPU-bound process. So multiprocessing is not a meaningful solution.
*update I finally found a demo ip camera with the same streaming interface as yours (I think). Using the streaming interface, it only makes a connection once and then reads from the stream of data as if it were a file to extract jpg image frames. With the code below, I grabbed for 2 seconds ==> 27 frames which I believe extrapolates to about 300k images in a 7 hour period.
If you want to get even more, you would move the image modification and file writing to a separate thread and have a worker doing that while the main thread just grabs from the stream and sends jpeg data to the worker.
import base64
from datetime import datetime
import httplib
import io
import os
import time
from PIL import ImageFont
from PIL import Image
from PIL import ImageDraw
wdir = "workdir"
stream_url = ''
username = ''
password = ''
def main():
time_count = 2
looper_stream(time_count)
def looper_stream(time_count):
h = httplib.HTTP(stream_url)
h.putrequest('GET', '/videostream.cgi')
h.putheader('Authorization', 'Basic %s' % base64.encodestring('%s:%s' % (username, password))[:-1])
h.endheaders()
errcode, errmsg, headers = h.getreply()
stream_file = h.getfile()
start = time.time()
end = start + time_count
while time.time() <= end:
now = datetime.now()
dte = str(now.day) + "-" + str(now.month) + "-" + str(now.year)
dte1 = str(now.hour) + "-" + str(now.minute) + "-" + str(now.second) + "." + str(now.microsecond)
cname = "Cam1-"
dnow = """Date: %s """ % dte
dnow1 = """Time: %s""" % dte1
# your camera may have a different streaming format
# but I think you can figure it out from the debug style below
source_name = stream_file.readline() # '--ipcamera'
content_type = stream_file.readline() # 'Content-Type: image/jpeg'
content_length = stream_file.readline() # 'Content-Length: 19565'
print 'confirm/adjust content (source?): ' + source_name
print 'confirm/adjust content (type?): ' + content_type
print 'confirm/adjust content (length?): ' + content_length
# find the beginning of the jpeg data BEFORE pulling the jpeg framesize
# there must be a more efficient way, but hopefully this is not too bad
b1 = b2 = b''
while True:
b1 = stream_file.read(1)
while b1 != chr(0xff):
b1 = stream_file.read(1)
b2 = stream_file.read(1)
if b2 == chr(0xd8):
break
# pull the jpeg data
framesize = int(content_length[16:])
jpeg_stripped = b''.join((b1, b2, stream_file.read(framesize - 2)))
# throw away the remaining stream data. Sorry I have no idea what it is
junk_for_now = stream_file.readline()
# convert directly to an Image instead of saving / reopening
# thanks to SO: http://stackoverflow.com/a/12020860/377366
image_as_file = io.BytesIO(jpeg_stripped)
image_as_pil = Image.open(image_as_file)
draw = ImageDraw.Draw(image_as_pil)
draw.text((0, 0), cname, fill="white")
draw.text((0, 10), dnow, fill="white")
draw.text((0, 20), dnow1, fill="white")
img_name = "Cam1-" + dte + dte1 + ".jpg"
img_path = os.path.join(wdir, img_name)
image_as_pil.save(img_path)
if __name__ == '__main__':
main()
*jpg capture below doesn't seem fast enough which is logical. making so many http requests would be slow for anything.
from datetime import datetime
import io
import threading
import os
import time
import urllib2
from PIL import ImageFont
from PIL import Image
from PIL import ImageDraw
wdir = "workdir"
def looper(time_count, loop_name):
start = time.time()
end = start + time_count
font = ImageFont.truetype("/usr/share/fonts/truetype/ttf-dejavu/DejaVuSans.ttf", 10)
while time.time() <= end:
now = datetime.now()
dte = str(now.day) + "-" + str(now.month) + "-" + str(now.year)
dte1 = str(now.hour) + "-" + str(now.minute) + "-" + str(now.second) + "." + str(now.microsecond)
cname = "Cam1-"
dnow = """Date: %s """ % dte
dnow1 = """Time: %s""" % dte1
image = urllib2.urlopen('http://(ip address)/snapshot.cgi?user=uname&pwd=password').read()
# convert directly to an Image instead of saving / reopening
# thanks to SO: http://stackoverflow.com/a/12020860/377366
image_as_file = io.BytesIO(image)
image_as_pil = Image.open(image_as_file)
draw = ImageDraw.Draw(image_as_pil)
draw_text = "\n".join((cname, dnow, dnow1))
draw.text((0, 0), draw_text, fill="white", font=font)
#draw.text((0, 0), cname, fill="white", font=font)
#draw.text((0, 10), dnow, fill="white", font=font)
#draw.text((0, 20), dnow1, fill="white", font=font)
img_name = "Cam1-" + dte + dte1 + "(" + loop_name + ").jpg"
img_path = os.path.join(wdir, img_name)
image_as_pil.save(img_path)
if __name__ == '__main__':
time_count = 5
threads = list()
for i in range(2):
name = str(i)
t = threading.Thread(target=looper, args=(time_count, name))
threads.append(p)
t.start()
for t in threads:
t.join()