I am making app for real time detection of some object and detecting when was last time it moved. I made python object detection and made it server so that it sends frames on mobile kivy app that is showing real time video from server and it shows when was last time it moved. Everything worken until i converted that app to android using buildozer, it gives me server timeout error. Here is server side :
import cv2
import numpy as np
import os
import tensorflow as tf
import socket
import pickle
import struct
from object_detection.utils import label_map_util
from object_detection.utils import visualization_utils as viz_utils
from object_detection.builders import model_builder
from object_detection.utils import config_util
@tf.function
def detect_fn(image):
image, shapes = detection_model.preprocess(image)
prediction_dict = detection_model.predict(image, shapes)
detections = detection_model.postprocess(prediction_dict, shapes)
return detections
server_socket = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
data_socket = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
host_name = socket.gethostname()
host_ip = socket.gethostbyname(host_name)
print('Host IP:', host_ip)
port = 777
port2= 555
socket_address = (host_ip,port)
data_address = (host_ip, port2)
server_socket.bind(socket_address)
data_socket.bind(data_address)
TF_RECORD_SCRIPT_NAME = 'generate_tfrecord.py'
LABEL_MAP_NAME = 'label_map.pbtxt'
paths = {
'CHECKPOINT_PATH': os.path.join('zavrseni modeli', 'model jaje'),
}
files = {
'PIPELINE_CONFIG':os.path.join('zavrseni modeli', 'model jaje', 'pipeline.config'),
'LABELMAP': os.path.join('zavrseni modeli', 'model jaje', 'label_map.pbtxt')
}
configs = config_util.get_configs_from_pipeline_file(files['PIPELINE_CONFIG'])
detection_model = model_builder.build(model_config=configs['model'], is_training=False)
ckpt = tf.compat.v2.train.Checkpoint(model=detection_model)
ckpt.restore(os.path.join(paths['CHECKPOINT_PATH'], 'ckpt-31')).expect_partial()
category_index = label_map_util.create_category_index_from_labelmap (files['LABELMAP'])
cap = cv2.VideoCapture(0)
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
server_socket.listen(5)
data_socket.listen(5)
print("LISTENING AT: ",socket_address)
print("LISTENING AT: ",data_address)
client_socket, addr = server_socket.accept()
client_data, addr2 = data_socket.accept()
print('GOT CONNECTION FROM: ', addr)
print('GOT CONNECTION FROM: ', addr2)
ko=0
while cap.isOpened():
ret, frame = cap.read()
image_np = np.array(frame)
if image_np is not None:
input_tensor = tf.convert_to_tensor(np.expand_dims(image_np, 0), dtype=tf.float32)
detections = detect_fn(input_tensor)
if detections is not None :
num_detections = int(detections.pop('num_detections'))
detections = {key: value[0, :num_detections].numpy()
for key, value in detections.items()}
detections['num_detections'] = num_detections
detections['detection_classes'] = detections['detection_classes'].astype(np.int64)
label_id_offset = 1
image_np_with_detections = image_np.copy()
viz_utils.visualize_boxes_and_labels_on_image_array(
image_np_with_detections,
detections['detection_boxes'],
detections['detection_classes'] + label_id_offset,
detections['detection_scores'],
category_index,
use_normalized_coordinates=True,
max_boxes_to_draw=1,
min_score_thresh=.7,
agnostic_mode=False)
a = pickle.dumps(image_np_with_detections)
message = struct.pack("Q", len(a)) + a
client_socket.sendall(message)
cv2.imshow("Detections",image_np_with_detections)
min_score_thresh = .7
threshold = 0.5
max_scores = max(detections['detection_scores'])
for i in range(num_detections):
box = detections['detection_boxes'][i]
score = detections['detection_scores'][i]
if score > .7:
ymin, xmin, ymax, xmax = box.tolist()
pos = ymin + ymax + xmax + xmin
print("POS:", pos)
print("KO: ",ko)
x = abs(pos-ko)
print(x)
if x>0.05:
print("Poslato")
client_data.send(str(5).encode())
ko = pos
else :
print("nije poslato")
ko = pos
else :
client_data.send(str(0).encode())
if cv2.waitKey(10) & 0xFF == ord('q'):
cap.release()
cv2.destroyAllWindows()
client_socket.close()
client_data.close()
break
Here is client, kivy side :
from kivymd.app import MDApp
from kivymd.uix.boxlayout import MDBoxLayout
from kivymd.uix.button import MDRaisedButton
from kivy.uix.image import Image
from kivymd.uix.label import MDLabel
from kivy.clock import Clock
from kivy.graphics.texture import Texture
from kivy import platform
import socket
import cv2
import pickle
import struct
from datetime import datetime
if platform=="android":
from android.permissions import request_permissions, Permission
request_permissions([Permission.INTERNET])
class Angelus(MDApp):
def build(self):
layout = MDBoxLayout(orientation='vertical')
self.text= "CONNECT"
self.image = Image()
layout.add_widget(self.image)
self.button = MDRaisedButton()
self.label = MDLabel(text=self.text,
adaptive_height = True,
halign='center', valign='middle')
layout.add_widget(self.label)
self.counter = 0
self.client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.data_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.host_name = socket.gethostname()
#self.host_ip = socket.gethostbyname(self.host_name)
self.host_ip = "192.168.56.1"
self.port = 777
self.port2 = 555
self.client_socket.connect((self.host_ip, self.port))
self.data_socket.connect((self.host_ip, self.port2))
self.data = b""
self.payload_size = struct.calcsize("Q")
Clock.schedule_interval(lambda dt: self.update_data(self.data_socket), 1/2)
Clock.schedule_interval(lambda dt: self.update_frame(self.client_socket), 1.0 / 30.0)
return layout
def update_frame(self, dt):
while len(self.data) < self.payload_size:
packet = self.client_socket.recv(4 * 1024)
if not packet: break
self.data += packet
packet_msg_size = self.data[:self.payload_size]
self.data = self.data[self.payload_size:]
msg_size = struct.unpack("Q", packet_msg_size)[0]
while len(self.data) < msg_size:
self.data += self.client_socket.recv(4 * 1024)
frame_data = self.data[:msg_size]
self.data = self.data[msg_size:]
frame = pickle.loads(frame_data)
buffer = cv2.flip(frame, 0).tobytes()
texture = Texture.create(size=(frame.shape[1], frame.shape[0]), colorfmt='bgr')
texture.blit_buffer(buffer, colorfmt='bgr', bufferfmt='ubyte')
self.image.texture = texture
def update_data(self, dt):
data2 = str(self.data_socket.recv(1024).decode())
self.counter= self.counter +1
print(data2[0])
now = datetime.now()
current_time = now.strftime("%H:%M:%S")
if (self.counter==1):
self.label.text="CONNECTED"
self.label.size_hint_x = None
self.label.width = self.label.texture_size[0]
data2 = []
elif data2[0] == "5":
self.label.text = "MOVED AT: " + current_time
self.label.size_hint_x = None
self.label.width = self.label.texture_size[0]
data2 = []
Angelus().run()
and here is buildozer spec file :
[app]
# (str) Title of your application
title = Angelius
# (str) Package name
package.name = Angelius
# (str) Package domain (needed for android/ios packaging)
package.domain = org.test
# (str) Source code where the main.py live
source.dir = .
# (list) Source files to include (let empty to include all the files)
source.include_exts = py,png,jpg,kv,atlas
# (list) List of inclusions using pattern matching
#source.include_patterns = assets/*,images/*.png
# (list) Source files to exclude (let empty to not exclude anything)
#source.exclude_exts = spec
# (list) List of directory to exclude (let empty to not exclude anything)
#source.exclude_dirs = tests, bin, venv
# (list) List of exclusions using pattern matching
# Do not prefix with './'
#source.exclude_patterns = license,images/*/*.jpg
# (str) Application versioning (method 1)
version = 0.1
# (str) Application versioning (method 2)
# version.regex = __version__ = ['"](.*)['"]
# version.filename = %(source.dir)s/main.py
# (list) Application requirements
# comma separated e.g. requirements = sqlite3,kivy
requirements = python3,kivy,kivymd,opencv,datetime
# (str) Custom source folders for requirements
# Sets custom source for any requirements with recipes
# requirements.source.kivy = ../../kivy
# (str) Presplash of the application
#presplash.filename = %(source.dir)s/data/presplash.png
# (str) Icon of the application
#icon.filename = %(source.dir)s/data/icon.png
# (str) Supported orientation (one of landscape, sensorLandscape, portrait or all)
orientation = portrait
# (list) List of service to declare
#services = NAME:ENTRYPOINT_TO_PY,NAME2:ENTRYPOINT2_TO_PY
#
# OSX Specific
#
#
# author = © Copyright Info
# change the major version of python used by the app
osx.python_version = 3
# Kivy version to use
osx.kivy_version = 1.9.1
#
# Android specific
#
# (bool) Indicate if the application should be fullscreen or not
fullscreen = 0
# (string) Presplash background color (for android toolchain)
# Supported formats are: #RRGGBB #AARRGGBB or one of the following names:
# red, blue, green, black, white, gray, cyan, magenta, yellow, lightgray,
# darkgray, grey, lightgrey, darkgrey, aqua, fuchsia, lime, maroon, navy,
# olive, purple, silver, teal.
#android.presplash_color = #FFFFFF
# (string) Presplash animation using Lottie format.
# see https://lottiefiles.com/ for examples and https://airbnb.design/lottie/
# for general documentation.
# Lottie files can be created using various tools, like Adobe After Effect or Synfig.
#android.presplash_lottie = "path/to/lottie/file.json"
# (str) Adaptive icon of the application (used if Android API level is 26+ at runtime)
#icon.adaptive_foreground.filename = %(source.dir)s/data/icon_fg.png
#icon.adaptive_background.filename = %(source.dir)s/data/icon_bg.png
# (list) Permissions
android.permissions = INTERNET
# (list) features (adds uses-feature -tags to manifest)
#android.features = android.hardware.usb.host
I used scrcpy and adb logcat -s python to see logs and gives me timeout error. I also tried on 2 diffirent phones same error, btw platform is android. Ip and ports are good as you can see, and i also thought that there is some error im my kivy app so i made one test app that just connects sockets and it was same error.
So error was that for someone reason i got 2 IPv4 address or like 2 ports and my phone can't 'see' one that host_ip = socket.gethostbyname(host_name) is giving but rather other one that i got using ipconfig in cmd. Now it works.