2 Point Head Tracking,
1 not moving referenz point (at your nacklace)
1 moving point on a wooden bar on your helmet
60 frames per second, stable
benötigte Hardware:
– PS3 Eye Cam ( very high framerates )
– PC/Laptop ( DualCore 2×1,8GHz, 4GB, Win7 )
– USB Serial Light Adapter
– Arduino Leonardo
– PS4
– evtl. USB Hub
– LED infrared, 2 pieces
– Resistors 100 Ohm, 2 pieces
– Batteries/Accus, AA 1.2V , 4 pieces
– Batterie Case with switch, 2 pieces
– helmet or similar else
– wooden bar or similar else
– duct tape
Hardware Hacks:
– PS3 Eye Cam – IR blocking Filter entfernen und Tageslicht blocking filter (Diskette) einsetzen
Software:
– PC/Laptop – Windows Treiber der PS3 Eye Cam
– PC/Laptop – Python 2.7.3
– PC/Laptop – openCV
– PC/Laptop – pyserial
– PC/Laptop – Arduino IDE
– Arduino Leonardo – extra sketch
Following script holds everything together:
– reading camera stream,
– analysing, finding blobs,
– calculating distances,
– calculating movement between current versus previous frame
– sending relative movement over USBSerialLight Adapter to ArduinoLeonardo
import cv2
import time
import numpy
# python C:\python_scripts\opencv\cam_vid_60fps_binarized_flipped_blobs_SERIAL.py
##########################################
# setting up serial communication
import serial
#import time
import os
"""
C:\Users\XXXXX>python -m serial.tools.list_ports -a
Usage: list_ports.py [options] []
list_ports.py: error: no such option: -a
"""
# C:\Python27\Lib\site-packages\serial\tools\list_ports.py <<< look for options.. discoverd -v ! write_to_path = "C:\\python_scripts\\py_serial\\actual_com_ports.txt" os.system("python -m serial.tools.list_ports -v > " + write_to_path)
fr = open( write_to_path , "rb")
frc = fr.read()
fr.close()
"""
COM14
desc: Arduino Uno (COM14)
hwid: USB VID:PID=2341:0001 SNR=55330343831351D03232
COM9
desc: Arduino Leonardo (COM9)
hwid: USB VID:PID=2341:8036
2 ports found
"""
frc = str(frc).split("\n")
com_dict = {}
COM_id = ""
for line in frc:
#print line
if line[0:3] == "COM":
COM_id = line.strip()
#print "COM_id: ",COM_id
com_dict[COM_id] = ""
if line.find("desc:") != -1:
line2 = line
line2 = line2.replace("desc: ","")
line2 = line2.replace("("+COM_id+")","")
line2 = line2.replace("\t","")
line2 = line2.strip()
#print "line2: >>",line2,"<<"
com_dict[COM_id] = line2
arduino_leonardo_COMport = ""
arduino_uno_COMport = ""
for e in com_dict:
print e, ": ", com_dict[e]
if com_dict[e].lower().find("leonardo")!=-1:
arduino_leonardo_COMport = e
if com_dict[e].lower().find("uno")!=-1:
arduino_uno_COMport = e
print "arduino_uno_COMport: ",arduino_uno_COMport
print "arduino_leonardo_COMport: ",arduino_leonardo_COMport
if 1==1:
#COMport = 14
#serial_port = COMport-1
#serial_port = "COM14"
#serial_port = "COM11"
#serial_port = arduino_leonardo_COMport
serial_port = arduino_uno_COMport
baudrate = 9600
connected = False
"""
ser = serial.Serial(port=serial_port,
baudrate=baudrate,
parity=serial.PARITY_NONE,
stopbits=serial.STOPBITS_TWO,
bytesize=serial.EIGHTBITS,
timeout=0,
xonxoff=0,
rtscts=0)
"""
ser = serial.Serial( serial_port, baudrate) # open first serial port
#ser.DtrEnable = "true";
print("initialising")
time.sleep(2) # waiting the initialization...
print ser.name # check which port was really used
# EOF setting up serial
######################################################
#
# python C:\python_scripts\opencv\cam_vid_60fps_binarized_flipped_blobs_SERIAL.py
# http://docs.opencv.org/trunk/doc/py_tutorials/py_gui/py_video_display/py_video_display.html
#cam = cv2.VideoCapture(0) # laptop eingebaute cam
#cam = cv2.VideoCapture(1) # wenn direct am laptop
cam = cv2.VideoCapture(1)
print "cam.isOpened(): ",cam.isOpened()
# - CV_CAP_PROP_FPS Frame rate.
#print "FPS: ", cam.get( 10 ) # ? ret
#print "x: ", cam.GetCaptureProperty( CV_CAP_PROP_FPS )
print "CV_CAP_PROP_FRAME_WIDTH: ", cam.get(3)
print "CV_CAP_PROP_FRAME_HEIGHT: ", cam.get(4)
print "CV_CAP_PROP_FPS: ", cam.get(5)
# cam.get(propId)
# propId integer 0 bis 18
if 1==11:
size = (int(cv2.GetCaptureProperty(cam , cv2.CV_CAP_PROP_FRAME_WIDTH)), int(cv2.GetCaptureProperty(cam , cv2.CV_CAP_PROP_FRAME_HEIGHT)))
print "size: ",size
"""
Property identifier. It can be one of the following:
0 CV_CAP_PROP_POS_MSEC Current position of the video file in milliseconds or video capture timestamp.
1 CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be decoded/captured next.
2 CV_CAP_PROP_POS_AVI_RATIO Relative position of the video file: 0 - start of the film, 1 - end of the film.
3 CV_CAP_PROP_FRAME_WIDTH Width of the frames in the video stream.
4 CV_CAP_PROP_FRAME_HEIGHT Height of the frames in the video stream.
5 CV_CAP_PROP_FPS Frame rate.
6 CV_CAP_PROP_FOURCC 4-character code of codec.
7 CV_CAP_PROP_FRAME_COUNT Number of frames in the video file.
8 CV_CAP_PROP_FORMAT Format of the Mat objects returned by retrieve() .
9 CV_CAP_PROP_MODE Backend-specific value indicating the current capture mode.
10 CV_CAP_PROP_BRIGHTNESS Brightness of the image (only for cameras).
11 CV_CAP_PROP_CONTRAST Contrast of the image (only for cameras).
12 CV_CAP_PROP_SATURATION Saturation of the image (only for cameras).
13 CV_CAP_PROP_HUE Hue of the image (only for cameras).
14 CV_CAP_PROP_GAIN Gain of the image (only for cameras).
15 CV_CAP_PROP_EXPOSURE Exposure (only for cameras).
16 CV_CAP_PROP_CONVERT_RGB Boolean flags indicating whether images should be converted to RGB.
17 CV_CAP_PROP_WHITE_BALANCE Currently not supported
18 CV_CAP_PROP_RECTIFICATION Rectification flag for stereo cameras (note: only supported by DC1394 v 2.x backend currently)
"""
if 1==11:
cam.set(3,320) # CV_CAP_PROP_FRAME_WIDTH
cam.set(4,240) # CV_CAP_PROP_FRAME_HEIGHT
cam.set(5,100) # fps
print "cam get fps: ", cam.get(5) # fps
# 'module' object has no attribute 'createBackgroundSubtractorMOG'
#fgbg = cv2.BackgroundSubtractorMOG()
#fgbg = cv2.BackgroundSubtractorMOG2()
counter_frames = 0
start_time = time.time()
fps = 0
# http://opencv-python-tutroals.readthedocs.org/en/latest/py_tutorials/py_imgproc/py_morphological_ops/py_morphological_ops.html
#kernel = numpy.ones((2,2),numpy.uint8)
#kernel = numpy.ones((5,5),numpy.uint8)
kernel = cv2.getStructuringElement(cv2.MORPH_RECT,(5,5))
#kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(5,5))
#kernel = cv2.getStructuringElement(cv2.MORPH_CROSS,(5,5))
#kernel = numpy.ones((10,10),numpy.uint8)
cv2__THRESH_BINARY = cv2.THRESH_BINARY
cv2__flip = cv2.flip
cv2__morphologyEx = cv2.morphologyEx
cv2__MORPH_OPEN = cv2.MORPH_OPEN
cv2__MORPH_CLOSE = cv2.MORPH_CLOSE
cv2__imshow = cv2.imshow
cv2__waitKey = cv2.waitKey
if 1==11:
flags = [i for i in dir(cv2) if i.startswith('COLOR_')]
for flag in flags:
if flag.find("RGB")!=-1:
print flag
# COLOR_RGB2GRAY
prev_frame_dist_x = 999999 #init value
prev_frame_dist_y = 999999 # init value
while(True):
ret, frame = cam.read()
#cv2.imshow('frame', frame)
# Our operations on the frame come here
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
#gray = cv2.cvtColor(frame, cv2.THRESH_BINARY)
tresh = 253
#tresh = 200
ret,frame = cv2.threshold(frame,tresh,255,cv2__THRESH_BINARY )
frame = cv2__flip(frame, 1) #
#print "frame.dtype: ",frame.dtype
# http://opencv-python-tutroals.readthedocs.org/en/latest/py_tutorials/py_imgproc/py_morphological_ops/py_morphological_ops.html
#frame = cv2.erode(frame,kernel,iterations = 1)
frame = cv2.dilate(frame,kernel,iterations = 2)
#frame = cv2.erode(frame,kernel,iterations = 3)
#frame = cv2__morphologyEx(frame, cv2__MORPH_OPEN, kernel) # erosion followed by dilation
#frame = cv2__morphologyEx(frame, cv2.MORPH_TOPHAT, kernel)
#frame = cv2.morphologyEx(frame, cv2.MORPH_GRADIENT, kernel) # does outline
cv2__imshow('frame', frame)
#print "frame.dtype: ",frame.dtype
#fgmask = fgbg.apply(frame)
#cv2.imshow('frame',fgmask)
#x = cv2.cvtColor(frame, cv2.CV_SHAPE_CROSS)
#cv2.imshow('frame', x)
#frame = cv2.convertTo(frame, CV_8U)
# http://opencvpython.blogspot.de/2012/04/contour-features.html
contours, _ = cv2.findContours(frame, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
contour_number = 0
contours_dict = {}
for c in contours:
rect = cv2.boundingRect(c)
# rect[2] width
# rect[3] height
# sucht nur grosse blobs, alles kleiner 10x10 wird verworfen
if rect[2] < 10 or rect[3] < 10: continue
contour_number = contour_number+1
print "contour_number: ",contour_number
#print cv2.contourArea(c)
print rect # x,y width,height
contours_dict[contour_number] = rect
print
if contour_number == 2:
con1_x = contours_dict[1][0]
con2_x = contours_dict[2][0]
diff_con1_x_con2_x = con1_x - con2_x
print "diff_con1_x_con2_x: ", diff_con1_x_con2_x
con1_y = contours_dict[1][1]
con2_y = contours_dict[2][1]
diff_con1_y_con2_y = con1_y - con2_y
print "diff_con1_y_con2_y: ", diff_con1_y_con2_y
if diff_con1_x_con2_x != 999999: # 999999 is starting init val
diff_prev_frame_x = diff_con1_x_con2_x - prev_frame_dist_x
print "diff_prev_frame_x: ",diff_prev_frame_x
# x negativ value = move to the right
# x positiv value = move to the left
diff_prev_frame_y = diff_con1_y_con2_y - prev_frame_dist_y
# y negativ value = move to bottom
# y positiv value = move to top
print "diff_prev_frame_y: ",diff_prev_frame_y
# send these values to arduino
# diff_prev_frame_x
# diff_prev_frame_y
# or manipulate them multiple 2 or progressive
diff_prev_frame_x = diff_prev_frame_x * -1
# ABSoluter wert, positiv.. d.h negativ number positiv machen
ABS_x = diff_con1_x_con2_x
if diff_con1_x_con2_x < 0:
ABS_x = diff_con1_x_con2_x * -1
ABS_y = diff_con1_y_con2_y
if diff_con1_y_con2_y < 0:
ABS_y = diff_con1_y_con2_y * -1
print "ABS_x: ",ABS_x
print "ABS_y: ",ABS_y
if ABS_x < 100:
diff_prev_frame_x = diff_prev_frame_x * 2 # 2
else:
diff_prev_frame_x = diff_prev_frame_x * 4
# 240 center point y
# 280
if ABS_y > 200 or ABS_y < 280: diff_prev_frame_y = diff_prev_frame_y * 4 # 4 else: diff_prev_frame_y = diff_prev_frame_y * 8 ser.write( str(diff_prev_frame_x) + "," + str(diff_prev_frame_y) + "\n" ) prev_frame_dist_x = diff_con1_x_con2_x prev_frame_dist_y = diff_con1_y_con2_y # now map the distances # to mouse movements # mouse movement # there is an posiblity to use absolute mouse via arduino, # http://forum.arduino.cc/index.php?topic=94140.0 # http://arduino.stackexchange.com/questions/3531/using-leonardo-and-getting-absolute-x-y-mouse-values # WE NEED DATA FROM PREV FRAME # CALC differences # these differences map to mouse moves #print rect.bx,rect.by #,c.bw,c.bh #frame = cv2.drawContours(frame, contours, -1, (0,255,0), 3) # to quit press key "q" if cv2__waitKey(1) & 0xFF == ord('q'): break if 1==1: counter_frames = counter_frames + 1 #print "counter_frames: ",counter_frames cur_time = time.time() #print "start_time: ",start_time #print "cur_time: ",cur_time diff_time = cur_time - start_time #print "diff_time: ",diff_time if diff_time > 5: # 5 sekunden
fps = counter_frames / 5 # frames pro sekunde
counter_frames = 0
start_time = time.time()
print "fps: ",fps
print "fps: ",fps
cam.release()
cv2.destroyAllWindows()
Sketch for Arduino Leonardo
/*
Serial1-Mouse-Control
The mouse movement is always relative.
This sketch reads Serial1(RxTx) and uses incoming data
to set the movement of the mouse (over Serial, Main USB Port).
WARNING: When you use the Mouse.move() command, the Arduino takes
over your mouse! Make sure you have control before you use the mouse commands.
2015-02
Ronny Gey
*/
#include
// read string
String mouse_x = "";
String mouse_y = "";
void setup() {
// http://arduino.cc/en/pmwiki.php?n=Reference/Serial
// http://arduino.cc/en/Main/ArduinoBoardLeonardo
// Serial not needed, this is used by mouse control...
// only for string tests, debugging
//Serial.begin(9600); // first Serial Port, main USB port... OR sending to PS4
//while (!Serial) {
// ; // wait for serial port to connect. Needed for Leonardo only
//}
// second Serial Port on Leonardo! pins Rx(Pin0)/Tx(Pin1) for receiving data from Laptop
Serial1.begin(9600);
while (!Serial1) {
; // wait for serial port to connect. Needed for Leonardo only
}
// initialize mouse control:
Mouse.begin();
}
void loop() {
// read Serial_1, data coming from Laptop
// send data only when you receive data:
// read RxTx
if (Serial1.available() > 0) {
// getting x,y from Laptop over Serial1
// Kommaseparated value: x,y\n
// -10,5
// 5,-10
// 10,10
mouse_x = Serial1.readStringUntil(',');
mouse_y = Serial1.readStringUntil('\n');
//Serial1.print("I received X,Y: ");
//Serial1.print(mouse_x);
//Serial1.print(" : ");
//Serial1.println(mouse_y);
//Syntax: Mouse.move(xVal, yPos, wheel);
Mouse.move(mouse_x.toInt(), mouse_y.toInt(), 0);
// http://arduino.cc/en/Tutorial/StringToIntExample
} // EOF serial1 available
/* only for debugging purpose
// send data only when you receive data:
if (Serial.available() > 0) {
// getting x,y from Laptop over Serial1
// comma separated value: x,y\n
// -10,5
// 5,-10
// 10,10
mouse_x = Serial.readStringUntil(',');
mouse_y = Serial.readStringUntil('\n');
Serial.print("I received X,Y: ");
Serial.print(mouse_x);
Serial.print(" : ");
Serial.println(mouse_y);
} // EOF serial available
*/
}
thats all
