Updated docs with section on upgrading to VS2017

PythonClient/PythonClient.pyproj

@@ -5,14 +5,14 @@
     <SchemaVersion>2.0</SchemaVersion>
     <ProjectGuid>e2049e20-b6dd-474e-8bca-1c8dc54725aa</ProjectGuid>
     <ProjectHome>.</ProjectHome>
-    <StartupFile>hello_car.py</StartupFile>
+    <StartupFile>cv_navigate.py</StartupFile>
     <SearchPath>
     </SearchPath>
     <WorkingDirectory>.</WorkingDirectory>
     <OutputPath>.</OutputPath>
     <Name>PythonClient</Name>
     <RootNamespace>PythonClient</RootNamespace>
-    <InterpreterId>Global|ContinuumAnalytics|Anaconda35-64</InterpreterId>
+    <InterpreterId>CondaEnv|CondaEnv|airsim</InterpreterId>
     <InterpreterVersion>
     </InterpreterVersion>
   </PropertyGroup>
@@ -35,6 +35,8 @@
       <SubType>Code</SubType>
     </Compile>
     <Compile Include="car_monitor.py" />
+    <Compile Include="cv_capture.py" />
+    <Compile Include="cv_navigate.py" />
     <Compile Include="disarm.py" />
     <Compile Include="land.py" />
     <Compile Include="pause_continue_car.py">
@@ -95,7 +97,7 @@
     <Compile Include="survey.py" />
   </ItemGroup>
   <ItemGroup>
-    <InterpreterReference Include="Global|ContinuumAnalytics|Anaconda35-64" />
+    <InterpreterReference Include="CondaEnv|CondaEnv|airsim" />
   </ItemGroup>
   <PropertyGroup>
     <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">10.0</VisualStudioVersion>

PythonClient/cv_capture.py

@@ -0,0 +1,52 @@
+# In settings.json first activate computer vision mode: 
+# https://github.com/Microsoft/AirSim/blob/master/docs/image_apis.md#computer-vision-mode
+
+from AirSimClient import *
+import pprint
+import tempfile
+
+pp = pprint.PrettyPrinter(indent=4)
+
+client = MultirotorClient()
+client.confirmConnection()
+
+AirSimClientBase.wait_key('Press any key to get camera parameters')
+for camera_id in range(2):
+    camera_info = client.getCameraInfo(camera_id)
+    print("CameraInfo %d: %s" % (camera_id, pp.pprint(camera_info)))
+
+AirSimClientBase.wait_key('Press any key to get images')
+tmp_dir = os.path.join(tempfile.gettempdir(), "airsim_drone")
+print ("Saving images to %s" % tmp_dir)
+try:
+    for n in range(3):
+        os.makedirs(os.path.join(tmp_dir, str(n)))
+except OSError:
+    if not os.path.isdir(tmp_dir):
+        raise
+
+for x in range(50): # do few times
+    #xn = 1 + x*5  # some random number
+    client.simSetPose(Pose(Vector3r(x, 0, -2), AirSimClientBase.toQuaternion(0, 0, 0)), True)
+    time.sleep(0.1)
+
+    responses = client.simGetImages([
+        ImageRequest(0, AirSimImageType.Scene),
+        ImageRequest(1, AirSimImageType.Scene),
+        ImageRequest(2, AirSimImageType.Scene)])
+
+    for i, response in enumerate(responses):
+        if response.pixels_as_float:
+            print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_float), pprint.pformat(response.camera_position)))
+            AirSimClientBase.write_pfm(os.path.normpath(os.path.join(tmp_dir, str(x) + "_" + str(i) + '.pfm')), AirSimClientBase.getPfmArray(response))
+        else:
+            print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_uint8), pprint.pformat(response.camera_position)))
+            AirSimClientBase.write_file(os.path.normpath(os.path.join(tmp_dir, str(i), str(x) + "_" + str(i) + '.png')), response.image_data_uint8)
+
+    pose = client.simGetPose()
+    pp.pprint(pose)
+
+    time.sleep(3)
+
+# currently reset() doesn't work in CV mode. Below is the workaround
+client.simSetPose(Pose(Vector3r(0, 0, 0), AirSimClientBase.toQuaternion(0, 0, 0)), True)

PythonClient/cv_navigate.py

@@ -0,0 +1,266 @@
+# In settings.json first activate computer vision mode: 
+# https://github.com/Microsoft/AirSim/blob/master/docs/image_apis.md#computer-vision-mode
+
+from AirSimClient import *
+import pprint
+import tempfile
+from math import *
+from scipy.misc import imsave
+
+from abc import ABC, abstractmethod
+
+#define abstract class to return next vector in the format (x,y,yaw)
+class AbstractClassGetNextVec(ABC):
+
+    @abstractmethod
+    def get_next_vec(self, depth, obj_sz, goal, pos):
+        print("Some implementation!")
+        return pos,yaw
+
+class ReactiveController(AbstractClassGetNextVec):
+    def get_next_vec(self, depth, obj_sz, goal, pos):
+        print("Some implementation!")
+        return
+
+class AvoidLeft(AbstractClassGetNextVec):
+
+    def __init__(self, hfov=radians(90), coll_thres=5, yaw=0, limit_yaw=5, step=0.1):
+        self.hfov = hfov
+        self.coll_thres = coll_thres
+        self.yaw = yaw
+        self.limit_yaw = limit_yaw
+        self.step = step
+
+    def get_next_vec(self, depth, obj_sz, goal, pos):
+
+        [h,w] = np.shape(depth)
+        [roi_h,roi_w] = compute_bb((h,w), obj_sz, self.hfov, self.coll_thres)
+
+        # compute vector, distance and angle to goal
+        t_vec, t_dist, t_angle = get_vec_dist_angle (goal, pos[:-1])
+
+        # compute box of interest
+        img2d_box = img2d[int((h-roi_h)/2):int((h+roi_h)/2),int((w-roi_w)/2):int((w+roi_w)/2)]
+
+        # scale by weight matrix (optional)
+        #img2d_box = np.multiply(img2d_box,w_mtx)
+
+        # detect collision
+        if (np.min(img2d_box) < coll_thres):
+            self.yaw = self.yaw - radians(self.limit_yaw)
+        else:
+            self.yaw = self.yaw + min (t_angle-self.yaw, radians(self.limit_yaw))
+
+        pos[0] = pos[0] + self.step*cos(self.yaw)
+        pos[1] = pos[1] + self.step*sin(self.yaw)
+
+        return pos, self.yaw,t_dist
+
+class AvoidLeftIgonreGoal(AbstractClassGetNextVec):
+
+    def __init__(self, hfov=radians(90), coll_thres=5, yaw=0, limit_yaw=5, step=0.1):
+        self.hfov = hfov
+        self.coll_thres = coll_thres
+        self.yaw = yaw
+        self.limit_yaw = limit_yaw
+        self.step = step
+
+    def get_next_vec(self, depth, obj_sz, goal, pos):
+
+        [h,w] = np.shape(depth)
+        [roi_h,roi_w] = compute_bb((h,w), obj_sz, self.hfov, self.coll_thres)
+
+        # compute box of interest
+        img2d_box = img2d[int((h-roi_h)/2):int((h+roi_h)/2),int((w-roi_w)/2):int((w+roi_w)/2)]
+
+        # detect collision
+        if (np.min(img2d_box) < coll_thres):
+            self.yaw = self.yaw - radians(self.limit_yaw)
+
+        pos[0] = pos[0] + self.step*cos(self.yaw)
+        pos[1] = pos[1] + self.step*sin(self.yaw)
+
+        return pos, self.yaw, 100
+
+class AvoidLeftRight(AbstractClassGetNextVec):
+    def get_next_vec(self, depth, obj_sz, goal, pos):
+        print("Some implementation!")
+        #Same as above but decide to go left or right based on average or some metric like that
+        return
+
+
+#compute resultant normalized vector, distance and angle
+def get_vec_dist_angle (goal, pos):
+    vec = np.array(goal - np.array(pos))
+    dist = math.sqrt(vec[0]**2 + vec[1]**2)
+    angle = math.atan2(vec[1],vec[0])
+    if angle > math.pi:
+        angle -= 2*math.pi
+    elif angle < -math.pi:
+        angle += 2*math.pi
+    return vec/dist, dist, angle
+
+def get_local_goal (v, pos, theta):
+    return goal
+
+#compute bounding box size
+def compute_bb(image_sz, obj_sz, hfov, distance):
+    vfov = hfov2vfov(hfov,image_sz)
+    box_h = ceil(obj_sz[0] * image_sz[0] / (math.tan(hfov/2)*distance*2))
+    box_w = ceil(obj_sz[1] * image_sz[1] / (math.tan(vfov/2)*distance*2))
+    return box_h, box_w
+
+#convert horizonal fov to vertical fov
+def hfov2vfov(hfov, image_sz):
+    aspect = image_sz[0]/image_sz[1]
+    vfov = 2*math.atan( tan(hfov/2) * aspect)
+    return vfov
+
+#matrix with all ones
+def equal_weight_mtx(roi_h,roi_w):
+    return np.ones((roi_h,roi_w))
+
+#matrix with max weight in center and decreasing linearly with distance from center
+def linear_weight_mtx(roi_h,roi_w):
+    w_mtx = np.ones((roi_h,roi_w))
+    for j in range(0,roi_w):
+        for i in range(j,roi_h-j):
+            w_mtx[j:roi_h-j,i:roi_w-i] = (j+1)
+    return w_mtx
+
+#matrix with max weight in center and decreasing quadratically with distance from center
+def square_weight_mtx(roi_h,roi_w):
+    w_mtx = np.ones((roi_h,roi_w))
+    for j in range(0,roi_w):
+        for i in range(j,roi_h-j):
+            w_mtx[j:roi_h-j,i:roi_w-i] = (j+1)*(j+1)
+    return w_mtx
+
+def print_stats(img):
+    print ('Avg: ',np.average(img))
+    print ('Min: ',np.min(img))
+    print ('Max: ',np.max(img))
+    print('Img Sz: ',np.size(img))
+
+def generate_depth_viz(img,thres=0):
+    if thres > 0:
+        img[img > thres] = thres
+    else:
+        img = np.reciprocal(img)
+    return img 
+
+def moveUAV(client,pos,yaw):
+    client.simSetPose(Pose(Vector3r(pos[0], pos[1], pos[2]), AirSimClientBase.toQuaternion(0, 0, yaw)), True) 
+
+
+pp = pprint.PrettyPrinter(indent=4)
+
+client = MultirotorClient()
+client.confirmConnection()
+
+tmp_dir = os.path.join(tempfile.gettempdir(), "airsim_drone")
+#print ("Saving images to %s" % tmp_dir)
+#AirSimClientBase.wait_key('Press any key to start')
+
+#Define start position, goal and size of UAV
+pos = [0,5,-1] #start position x,y,z
+goal = [120,0] #x,y
+uav_size = [0.29*3,0.98*2] #height:0.29 x width:0.98 - allow some tolerance
+
+#Define parameters and thresholds
+hfov = radians(90)
+coll_thres = 5
+yaw = 0
+limit_yaw = 5
+step = 0.1
+
+responses = client.simGetImages([
+    ImageRequest(1, AirSimImageType.DepthPlanner, True)])
+response = responses[0]
+
+#initial position
+moveUAV(client,pos,yaw)
+
+#predictControl = AvoidLeftIgonreGoal(hfov, coll_thres, yaw, limit_yaw, step)
+predictControl = AvoidLeft(hfov, coll_thres, yaw, limit_yaw, step)
+
+for z in range(10000): # do few times
+
+    #time.sleep(1)
+
+    # get response
+    responses = client.simGetImages([
+        ImageRequest(1, AirSimImageType.DepthPlanner, True)])
+    response = responses[0]
+
+    # get numpy array
+    img1d = response.image_data_float
+
+    # reshape array to 2D array H X W
+    img2d = np.reshape(img1d,(response.height, response.width)) 
+
+    [pos,yaw,target_dist] = predictControl.get_next_vec(img2d, uav_size, goal, pos)
+    moveUAV(client,pos,yaw)
+
+    if (target_dist < 1):
+        print('Target reached.')
+        AirSimClientBase.wait_key('Press any key to continue')
+        break
+
+    # write to png 
+    #imsave(os.path.normpath(os.path.join(tmp_dir, "depth_" + str(z) + '.png')), generate_depth_viz(img2d,5))
+
+    #pose = client.simGetPose()
+    #pp.pprint(pose)
+    #time.sleep(5)
+
+# currently reset() doesn't work in CV mode. Below is the workaround
+client.simSetPose(Pose(Vector3r(0, 0, 0), AirSimClientBase.toQuaternion(0, 0, 0)), True)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+#################### OLD CODE
+#    timer = 0
+#    time_obs = 50
+#    bObstacle = False
+
+#    if (bObstacle):
+#        timer = timer + 1
+#        if timer > time_obs:
+#            bObstacle = False
+#            timer = 0
+#    else:
+#        yaw = target_angle
+
+#    print (target_angle,target_vec,target_dist,x,y,goal[0],goal[1])
+
+
+#    if (np.average(img2d_box) < coll_thres):
+#        img2d_box_l = img2d_box = img2d[int((h-roi_h)/2):int((h+roi_h)/2),int((w-roi_w)/2)-50:int((w+roi_w)/2)-50]
+#        img2d_box_r = img2d_box = img2d[int((h-roi_h)/2):int((h+roi_h)/2),int((w-roi_w)/2)+50:int((w+roi_w)/2)+50]
+#        img2d_box_l_avg = np.average(np.multiply(img2d_box_l,w_mtx))
+#        img2d_box_r_avg = np.average(np.multiply(img2d_box_r,w_mtx))
+#        print('left: ', img2d_box_l_avg)
+#        print('right: ', img2d_box_r_avg)
+#        if img2d_box_l_avg > img2d_box_r_avg:
+#            ##Go LEFT
+#            #y_offset = y_offset-1
+#            yaw = yaw - radians(10)
+#            bObstacle = True
+#        else:
+#            ##Go RIGHT
+#            #y_offset = y_offset+1
+#            yaw = yaw + radians(10)
+#            bObstacle = true
+#        print('yaw: ', yaw)

docs/build_windows.md

@@ -33,3 +33,15 @@ By default, AirSim uses its own built-in firmware called [simple_flight](simple_
 #### I made changes in Visual Studio but there is no effect
 
 Sometimes the Unreal + VS build system doesn't recompile if you make changes to only header files. To ensure a recompile, make some Unreal based cpp file "dirty" like AirSimGameMode.cpp.
+
+####Upgrading to Visual Studio 2017
+Running serveral versions of VS can lead to issues when compiling UE projects. One problem that may arise is that UE will try to compile with an older version of VS which may or may not work. There are two settings in Unreal, one for for the engine and one for the project, to adjust the version of VS to be used.
+1. Edit -> Editor preferences -> General -> Source code
+2. Edit -> Project Settings -> Platforms -> Windows -> Toolchain ->CompilerVersion
+
+In some cases, these settings will still not lead to the desired result and errors such as the following might be produced: LINK : fatal error LNK1181: cannot open input file 'ws2_32.lib'
+
+To resolve such issues the following procedure can be applied:
+1. Uninstall all old versions of VS using the [VisualStudioUninstaller](https://github.com/Microsoft/VisualStudioUninstaller/releases)
+2. Repair/Install VS2017
+3. Restart machine and install Epic launcher and desired version of the engine

docs/image_apis.md

@@ -129,7 +129,7 @@ For a more complete ready to run sample code please see [sample code in AirSimCl
 The camera ID 0 to 4 corresponds to center front, left front, right front, driver head, center rear respectively.
 
 ### Multirotor
-The camera ID 0 to 4 corresponds to center front, left front, right front, center downward, center rear respectively.
+The camera ID 0 to 4 corresponds to center front, right front, left front, center downward, center rear respectively.
 
 ## "Computer Vision" Mode