Skip to content

Commit ee41cc7

Browse files
sytelussytelus
committed
1 parent 32a7a44 commit ee41cc7

File tree

5 files changed

+378
-4
lines changed

5 files changed

+378
-4
lines changed

‎PythonClient/PythonClient.pyproj‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,12 @@
4141
<Compile Include="car\setup_path.py">
4242
<SubType>Code</SubType>
4343
</Compile>
44+
<Compile Include="computer_vision\cv_capture.py">
45+
<SubType>Code</SubType>
46+
</Compile>
47+
<Compile Include="computer_vision\cv_navigate.py">
48+
<SubType>Code</SubType>
49+
</Compile>
4450
<Compile Include="computer_vision\getpos.py" />
4551
<Compile Include="computer_vision\ground_truth.py">
4652
<SubType>Code</SubType>

‎PythonClient/computer_vision/cv_capture.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# In settings.json first activate computer vision mode:
2+
# https://github.com/Microsoft/AirSim/blob/master/docs/image_apis.md#computer-vision-mode
3+
4+
import setup_path
5+
import airsim
6+
7+
import pprint
8+
import tempfile
9+
import os
10+
import time
11+
12+
pp = pprint.PrettyPrinter(indent=4)
13+
14+
client = airsim.VehicleClient()
15+
16+
airsim.wait_key('Press any key to get camera parameters')
17+
for camera_id in range(2):
18+
camera_info = client.simGetCameraInfo(str(camera_id))
19+
print("CameraInfo %d: %s" % (camera_id, pp.pprint(camera_info)))
20+
21+
airsim.wait_key('Press any key to get images')
22+
tmp_dir = os.path.join(tempfile.gettempdir(), "airsim_drone")
23+
print ("Saving images to %s" % tmp_dir)
24+
try:
25+
for n in range(3):
26+
os.makedirs(os.path.join(tmp_dir, str(n)))
27+
except OSError:
28+
if not os.path.isdir(tmp_dir):
29+
raise
30+
31+
for x in range(50): # do few times
32+
#xn = 1 + x*5 # some random number
33+
client.simSetVehiclePose(airsim.Pose(airsim.Vector3r(x, 0, -2), airsim.to_quaternion(0, 0, 0)), True)
34+
time.sleep(0.1)
35+
36+
responses = client.simGetImages([
37+
airsim.ImageRequest("0", airsim.ImageType.Scene),
38+
airsim.ImageRequest("1", airsim.ImageType.Scene),
39+
airsim.ImageRequest("2", airsim.ImageType.Scene)])
40+
41+
for i, response in enumerate(responses):
42+
if response.pixels_as_float:
43+
print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_float), pprint.pformat(response.camera_position)))
44+
airsim.write_pfm(os.path.normpath(os.path.join(tmp_dir, str(x) + "_" + str(i) + '.pfm')), airsim.get_pfm_array(response))
45+
else:
46+
print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_uint8), pprint.pformat(response.camera_position)))
47+
airsim.write_file(os.path.normpath(os.path.join(tmp_dir, str(i), str(x) + "_" + str(i) + '.png')), response.image_data_uint8)
48+
49+
pose = client.simGetVehiclePose()
50+
pp.pprint(pose)
51+
52+
time.sleep(3)
53+
54+
# currently reset() doesn't work in CV mode. Below is the workaround
55+
client.simSetVehiclePose(airsim.Pose(airsim.Vector3r(0, 0, 0), airsim.to_quaternion(0, 0, 0)), True)

‎PythonClient/computer_vision/cv_navigate.py‎

Lines changed: 273 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,273 @@
1+
2+
# In settings.json first activate computer vision mode:
3+
# https://github.com/Microsoft/AirSim/blob/master/docs/image_apis.md#computer-vision-mode
4+
5+
import setup_path
6+
import airsim
7+
8+
import numpy as np
9+
import time
10+
import os
11+
import pprint
12+
import tempfile
13+
import math
14+
from math import *
15+
from scipy.misc import imsave
16+
17+
from abc import ABC, abstractmethod
18+
19+
#define abstract class to return next vector in the format (x,y,yaw)
20+
class AbstractClassGetNextVec(ABC):
21+
22+
@abstractmethod
23+
def get_next_vec(self, depth, obj_sz, goal, pos):
24+
print("Some implementation!")
25+
return pos,yaw
26+
27+
class ReactiveController(AbstractClassGetNextVec):
28+
def get_next_vec(self, depth, obj_sz, goal, pos):
29+
print("Some implementation!")
30+
return
31+
32+
class AvoidLeft(AbstractClassGetNextVec):
33+
34+
def __init__(self, hfov=radians(90), coll_thres=5, yaw=0, limit_yaw=5, step=0.1):
35+
self.hfov = hfov
36+
self.coll_thres = coll_thres
37+
self.yaw = yaw
38+
self.limit_yaw = limit_yaw
39+
self.step = step
40+
41+
def get_next_vec(self, depth, obj_sz, goal, pos):
42+
43+
[h,w] = np.shape(depth)
44+
[roi_h,roi_w] = compute_bb((h,w), obj_sz, self.hfov, self.coll_thres)
45+
46+
# compute vector, distance and angle to goal
47+
t_vec, t_dist, t_angle = get_vec_dist_angle (goal, pos[:-1])
48+
49+
# compute box of interest
50+
img2d_box = img2d[int((h-roi_h)/2):int((h+roi_h)/2),int((w-roi_w)/2):int((w+roi_w)/2)]
51+
52+
# scale by weight matrix (optional)
53+
#img2d_box = np.multiply(img2d_box,w_mtx)
54+
55+
# detect collision
56+
if (np.min(img2d_box) < coll_thres):
57+
self.yaw = self.yaw - radians(self.limit_yaw)
58+
else:
59+
self.yaw = self.yaw + min (t_angle-self.yaw, radians(self.limit_yaw))
60+
61+
pos[0] = pos[0] + self.step*cos(self.yaw)
62+
pos[1] = pos[1] + self.step*sin(self.yaw)
63+
64+
return pos, self.yaw,t_dist
65+
66+
class AvoidLeftIgonreGoal(AbstractClassGetNextVec):
67+
68+
def __init__(self, hfov=radians(90), coll_thres=5, yaw=0, limit_yaw=5, step=0.1):
69+
self.hfov = hfov
70+
self.coll_thres = coll_thres
71+
self.yaw = yaw
72+
self.limit_yaw = limit_yaw
73+
self.step = step
74+
75+
def get_next_vec(self, depth, obj_sz, goal, pos):
76+
77+
[h,w] = np.shape(depth)
78+
[roi_h,roi_w] = compute_bb((h,w), obj_sz, self.hfov, self.coll_thres)
79+
80+
# compute box of interest
81+
img2d_box = img2d[int((h-roi_h)/2):int((h+roi_h)/2),int((w-roi_w)/2):int((w+roi_w)/2)]
82+
83+
# detect collision
84+
if (np.min(img2d_box) < coll_thres):
85+
self.yaw = self.yaw - radians(self.limit_yaw)
86+
87+
pos[0] = pos[0] + self.step*cos(self.yaw)
88+
pos[1] = pos[1] + self.step*sin(self.yaw)
89+
90+
return pos, self.yaw, 100
91+
92+
class AvoidLeftRight(AbstractClassGetNextVec):
93+
def get_next_vec(self, depth, obj_sz, goal, pos):
94+
print("Some implementation!")
95+
#Same as above but decide to go left or right based on average or some metric like that
96+
return
97+
98+
99+
#compute resultant normalized vector, distance and angle
100+
def get_vec_dist_angle (goal, pos):
101+
vec = np.array(goal - np.array(pos))
102+
dist = math.sqrt(vec[0]**2 + vec[1]**2)
103+
angle = math.atan2(vec[1],vec[0])
104+
if angle > math.pi:
105+
angle -= 2*math.pi
106+
elif angle < -math.pi:
107+
angle += 2*math.pi
108+
return vec/dist, dist, angle
109+
110+
def get_local_goal (v, pos, theta):
111+
return goal
112+
113+
#compute bounding box size
114+
def compute_bb(image_sz, obj_sz, hfov, distance):
115+
vfov = hfov2vfov(hfov,image_sz)
116+
box_h = ceil(obj_sz[0] * image_sz[0] / (math.tan(hfov/2)*distance*2))
117+
box_w = ceil(obj_sz[1] * image_sz[1] / (math.tan(vfov/2)*distance*2))
118+
return box_h, box_w
119+
120+
#convert horizonal fov to vertical fov
121+
def hfov2vfov(hfov, image_sz):
122+
aspect = image_sz[0]/image_sz[1]
123+
vfov = 2*math.atan( tan(hfov/2) * aspect)
124+
return vfov
125+
126+
#matrix with all ones
127+
def equal_weight_mtx(roi_h,roi_w):
128+
return np.ones((roi_h,roi_w))
129+
130+
#matrix with max weight in center and decreasing linearly with distance from center
131+
def linear_weight_mtx(roi_h,roi_w):
132+
w_mtx = np.ones((roi_h,roi_w))
133+
for j in range(0,roi_w):
134+
for i in range(j,roi_h-j):
135+
w_mtx[j:roi_h-j,i:roi_w-i] = (j+1)
136+
return w_mtx
137+
138+
#matrix with max weight in center and decreasing quadratically with distance from center
139+
def square_weight_mtx(roi_h,roi_w):
140+
w_mtx = np.ones((roi_h,roi_w))
141+
for j in range(0,roi_w):
142+
for i in range(j,roi_h-j):
143+
w_mtx[j:roi_h-j,i:roi_w-i] = (j+1)*(j+1)
144+
return w_mtx
145+
146+
def print_stats(img):
147+
print ('Avg: ',np.average(img))
148+
print ('Min: ',np.min(img))
149+
print ('Max: ',np.max(img))
150+
print('Img Sz: ',np.size(img))
151+
152+
def generate_depth_viz(img,thres=0):
153+
if thres > 0:
154+
img[img > thres] = thres
155+
else:
156+
img = np.reciprocal(img)
157+
return img
158+
159+
def moveUAV(client,pos,yaw):
160+
client.simSetVehiclePose(airsim.Pose(airsim.Vector3r(pos[0], pos[1], pos[2]), airsim.to_quaternion(0, 0, yaw)), True)
161+
162+
163+
pp = pprint.PrettyPrinter(indent=4)
164+
165+
client = airsim.VehicleClient()
166+
client.confirmConnection()
167+
168+
tmp_dir = os.path.join(tempfile.gettempdir(), "airsim_drone")
169+
#print ("Saving images to %s" % tmp_dir)
170+
#airsim.wait_key('Press any key to start')
171+
172+
#Define start position, goal and size of UAV
173+
pos = [0,5,-1] #start position x,y,z
174+
goal = [120,0] #x,y
175+
uav_size = [0.29*3,0.98*2] #height:0.29 x width:0.98 - allow some tolerance
176+
177+
#Define parameters and thresholds
178+
hfov = radians(90)
179+
coll_thres = 5
180+
yaw = 0
181+
limit_yaw = 5
182+
step = 0.1
183+
184+
responses = client.simGetImages([
185+
airsim.ImageRequest("1", airsim.ImageType.DepthPlanner, True)])
186+
response = responses[0]
187+
188+
#initial position
189+
moveUAV(client,pos,yaw)
190+
191+
#predictControl = AvoidLeftIgonreGoal(hfov, coll_thres, yaw, limit_yaw, step)
192+
predictControl = AvoidLeft(hfov, coll_thres, yaw, limit_yaw, step)
193+
194+
for z in range(10000): # do few times
195+
196+
#time.sleep(1)
197+
198+
# get response
199+
responses = client.simGetImages([
200+
airsim.ImageRequest("1", airsim.ImageType.DepthPlanner, True)])
201+
response = responses[0]
202+
203+
# get numpy array
204+
img1d = response.image_data_float
205+
206+
# reshape array to 2D array H X W
207+
img2d = np.reshape(img1d,(response.height, response.width))
208+
209+
[pos,yaw,target_dist] = predictControl.get_next_vec(img2d, uav_size, goal, pos)
210+
moveUAV(client,pos,yaw)
211+
212+
if (target_dist < 1):
213+
print('Target reached.')
214+
airsim.wait_key('Press any key to continue')
215+
break
216+
217+
# write to png
218+
#imsave(os.path.normpath(os.path.join(tmp_dir, "depth_" + str(z) + '.png')), generate_depth_viz(img2d,5))
219+
220+
#pose = client.simGetPose()
221+
#pp.pprint(pose)
222+
#time.sleep(5)
223+
224+
# currently reset() doesn't work in CV mode. Below is the workaround
225+
client.simSetVehiclePose(airsim.Pose(airsim.Vector3r(0, 0, 0), airsim.to_quaternion(0, 0, 0)), True)
226+
227+
228+
229+
230+
231+
232+
233+
234+
235+
236+
237+
238+
239+
240+
#################### OLD CODE
241+
# timer = 0
242+
# time_obs = 50
243+
# bObstacle = False
244+
245+
# if (bObstacle):
246+
# timer = timer + 1
247+
# if timer > time_obs:
248+
# bObstacle = False
249+
# timer = 0
250+
# else:
251+
# yaw = target_angle
252+
253+
# print (target_angle,target_vec,target_dist,x,y,goal[0],goal[1])
254+
255+
256+
# if (np.average(img2d_box) < coll_thres):
257+
# img2d_box_l = img2d_box = img2d[int((h-roi_h)/2):int((h+roi_h)/2),int((w-roi_w)/2)-50:int((w+roi_w)/2)-50]
258+
# img2d_box_r = img2d_box = img2d[int((h-roi_h)/2):int((h+roi_h)/2),int((w-roi_w)/2)+50:int((w+roi_w)/2)+50]
259+
# img2d_box_l_avg = np.average(np.multiply(img2d_box_l,w_mtx))
260+
# img2d_box_r_avg = np.average(np.multiply(img2d_box_r,w_mtx))
261+
# print('left: ', img2d_box_l_avg)
262+
# print('right: ', img2d_box_r_avg)
263+
# if img2d_box_l_avg > img2d_box_r_avg:
264+
# ##Go LEFT
265+
# #y_offset = y_offset-1
266+
# yaw = yaw - radians(10)
267+
# bObstacle = True
268+
# else:
269+
# ##Go RIGHT
270+
# #y_offset = y_offset+1
271+
# yaw = yaw + radians(10)
272+
# bObstacle = true
273+
# print('yaw: ', yaw)

‎docs/build_windows.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,15 @@ By default, AirSim uses its own built-in firmware called [simple_flight](simple_
3333
#### I made changes in Visual Studio but there is no effect
3434

3535
Sometimes the Unreal + VS build system doesn't recompile if you make changes to only header files. To ensure a recompile, make some Unreal based cpp file "dirty" like AirSimGameMode.cpp.
36+
37+
#### Unreal still uses VS2015 or I'm getting some link error
38+
Running serveral versions of VS can lead to issues when compiling UE projects. One problem that may arise is that UE will try to compile with an older version of VS which may or may not work. There are two settings in Unreal, one for for the engine and one for the project, to adjust the version of VS to be used.
39+
1. Edit -> Editor preferences -> General -> Source code
40+
2. Edit -> Project Settings -> Platforms -> Windows -> Toolchain ->CompilerVersion
41+
42+
In some cases, these settings will still not lead to the desired result and errors such as the following might be produced: LINK : fatal error LNK1181: cannot open input file 'ws2_32.lib'
43+
44+
To resolve such issues the following procedure can be applied:
45+
1. Uninstall all old versions of VS using the [VisualStudioUninstaller](https://github.com/Microsoft/VisualStudioUninstaller/releases)
46+
2. Repair/Install VS2017
47+
3. Restart machine and install Epic launcher and desired version of the engine

0 commit comments

Comments
 (0)