Advanced Lane Finding - Project 4 of Udacity's Self-Driving Car Nanodegree¶
In this project the task is to detect the lane lines in a video using several filter techniques such as sobel based edge detections or HLS color space conversions and then to threshold these to detect the most likely lane pixels.
After the highlighting of the pixels the lane area is detected using a sliding window technique. Afterwards this the curvature is calculated by calculating the lane's polynomial and it's radius as well the offset of the car by calculating the relative position between the lane lines.
Before all of this can be done correctly the video images still need to be freed of theirs natural radial distortion by calculating the distortion using a set of chessboard images provided. While the lane detection binary masks are still applied in the perspective, undistorted version the curvature needs to be calculated in a top down view which is generated by warping a trapezoid area of the street into rectangular one. The lower part of this area is then internally scaled to the typical US highway width of 3.7 meters for the size calculations.
1. Camera Calibration¶
At first we need to calibrate the camera to remove it's original very intensive radial distortion.
To do so we use OpenCVs calibrate camera functions which use a couple of chessboard images. With the help of these (and known that a chessboard's lines are straight) it can calculate the distortion matrix.
HTML("""
<video width="960" height="540" controls>
<source src="{0}">
</video>
""".format('test_videos_output/combined.mp4'))
%run AdvLaneCamera.py
camera = AdvCamera()
camera.chessboard_calibrate_camera()
camera.save_to_pickle()
2. Distortion Correction¶
%run AdvLaneHelper.py
lane_helper = AdvLaneHelper(camera=camera)
images = glob.glob('camera_cal/calibration*.jpg')
index = 0
col_count = 3
row_count = 7
fig = plt.figure(figsize=(16,32))
# Step through the list and search for chessboard corners
for fname in images:
img = lane_helper.load_and_undistort(fname)
sp = fig.add_subplot(row_count, col_count, index+1)
plt.imshow(img)
index += 1
plt.show()
3. Example image inspection¶
example_images = lane_helper.get_example_images()
org_example_images = example_images
example_images = example_images[0:4]
col_count = 2
row_count = 8
fig = plt.figure(figsize=(25,60))
index = 0
example_image = None
# Step through the list and search for chessboard corners
for fname in example_images:
img = cv2.imread(fname)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
sp = fig.add_subplot(row_count, col_count, index*2+1)
plt.title("Distorted")
plt.imshow(img)
img = camera.undistort(img)
sp = fig.add_subplot(row_count, col_count, index*2+2)
plt.title("Undistorted")
plt.imshow(img)
index += 1
plt.show()
4. Perspective correction¶
No we remove the perspective and create a top down view of the most interesting area around the centers to later be able to search for straight lines, vertical lines.
%run AdvLanePerspectiveTransform.py
example_image = lane_helper.load_and_undistort(org_example_images[6])
perspective_transform = LanePerspectiveTransform(example_image)
# paint trapez into the image
image_copy = np.copy(example_image)
for index in range(4):
pa = perspective_transform.org_src[index]
pb = perspective_transform.org_src[(index+1)%4]
cv2.line(image_copy, pa, pb, (255,0,0), 4)
fig = plt.figure(figsize=(20,20))
plt.imshow(image_copy)
plt.show()
warped = perspective_transform.transform_perspective_top(example_image)
fig = plt.figure(figsize=(20,20))
plt.imshow(warped)
from moviepy.editor import VideoFileClip
from IPython.display import HTML
project_video = "project_video.mp4"
HTML("""
<video width="960" height="540" controls>
<source src="{0}">
</video>
""".format(project_video))
Video of the project video after distortion and perspective correction¶
def process_image(image):
undistorted = camera.undistort(image)
warped = perspective_transform.transform_perspective_top(undistorted)
return warped
from_above_video = 'test_videos_output/from_above.mp4'
white_output = from_above_video
clip1 = VideoFileClip(project_video)
white_clip = clip1.fl_image(process_image)
%time white_clip.write_videofile(white_output, audio=False)
HTML("""
<video width="960" height="540" controls>
<source src="{0}">
</video>
""".format(from_above_video))
5. Highlighting lanes¶
import ntpath
col_count = 2
row_count = 8
fig = plt.figure(figsize=(20,60))
index = 0
# Step through the list and search for chessboard corners
for fname in example_images:
img = lane_helper.load_and_undistort(fname)
sp = fig.add_subplot(row_count, col_count, index+1)
plt.title(ntpath.basename(fname))
warped = perspective_transform.transform_perspective_top(img)
plt.imshow(warped, 'gray')
index += 1
plt.show()
%run AdvLaneThresher.py
thresher = AdvLaneThresher()
col_count = 2
row_count = 8
fig = plt.figure(figsize=(20,60))
index = 0
# Step through the list and search for chessboard corners
for fname in example_images:
img = lane_helper.load_and_undistort(fname)
img, binmask = thresher.sobel_mag_mask(img)
warped = perspective_transform.transform_perspective_top(binmask)
sp = fig.add_subplot(row_count, col_count, index+1)
plt.title(ntpath.basename(fname))
plt.imshow(warped, 'gray')
index += 1
plt.show()
%run AdvLaneThresher.py
thresher = AdvLaneThresher()
col_count = 2
row_count = 8
fig = plt.figure(figsize=(20,60))
index = 0
# Step through the list and search for chessboard corners
for fname in example_images:
img = lane_helper.load_and_undistort(fname)
img, binmask = thresher.sobel_dir_mask(img)
warped = perspective_transform.transform_perspective_top(binmask)
sp = fig.add_subplot(row_count, col_count, index+1)
plt.title(ntpath.basename(fname))
plt.imshow(warped, 'gray')
index += 1
plt.show()
%run AdvLaneThresher.py
thresher = AdvLaneThresher()
col_count = 2
row_count = 8
fig = plt.figure(figsize=(20,60))
index = 0
# Step through the list and search for chessboard corners
for fname in example_images:
img = lane_helper.load_and_undistort(fname)
img, binmask = thresher.hls_threshold_mask(img)
warped = perspective_transform.transform_perspective_top(binmask)
sp = fig.add_subplot(row_count, col_count, index+1)
plt.title(ntpath.basename(fname))
plt.imshow(warped, 'gray')
index += 1
plt.show()
%run AdvLaneThresher.py
thresher = AdvLaneThresher()
col_count = 2
row_count = 8
fig = plt.figure(figsize=(20,60))
index = 0
# Step through the list and search for chessboard corners
for fname in example_images:
img = lane_helper.load_and_undistort(fname)
img, binary = thresher.sobel_abs_mask_x(img)
warped = perspective_transform.transform_perspective_top(binary)
sp = fig.add_subplot(row_count, col_count, index+1)
plt.title(ntpath.basename(fname))
plt.imshow(warped, 'gray')
index += 1
plt.show()
%run AdvLaneThresher.py
thresher = AdvLaneThresher()
col_count = 2
row_count = 8
fig = plt.figure(figsize=(20,60))
index = 0
# Step through the list and search for chessboard corners
for fname in example_images:
img = lane_helper.load_and_undistort(fname)
img = thresher.create_binary_mask(img)
warped = perspective_transform.transform_perspective_top(img)
sp = fig.add_subplot(row_count, col_count, index+1)
plt.title(ntpath.basename(fname))
plt.imshow(warped, 'gray')
index += 1
plt.show()
6. Build histograms to highlight likely lane positions¶
The histograms below show where there are very likely positions for the left and right lane by summing up the positive matches in the binary mask for each column
col_count = 2
row_count = 8
fig = plt.figure(figsize=(20,60))
index = 0
# Step through the list and search for chessboard corners
for fname in example_images:
img = lane_helper.load_and_undistort(fname)
img = thresher.create_binary_mask(img)
warped = perspective_transform.transform_perspective_top(img)
sp = fig.add_subplot(row_count, col_count, index+1)
plt.title(ntpath.basename(fname))
histogram = np.sum(warped[img.shape[0]//2:,:], axis=0)
plt.plot(histogram)
index += 1
plt.show()
7. Detection of the lanes¶
With a sliding window we now try to detect which way each line takes along the top view image.
In case of a single image we assume the left and right lane to start at the histograms peaks.
When processing several images we calculate the likeliness that the new histogram value is valid and otherwise use a more likely starting position from a previous frame.
%run AdvLaneFinder.py
lane_finder = LaneFinder(camera, perspective_transform, thresher)
for cur_fn in example_images:
img = cv2.imread(cur_fn)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
lane_finder.clear_history()
out_img, cam_img, persp = lane_finder.find_lanes_using_window(img)
fig = plt.figure(figsize=(12,8))
plt.title(cur_fn)
plt.imshow(out_img)
plt.xlim(0, 1280)
plt.ylim(720, 0)
plt.show()
%run AdvLaneFinder.py
lane_finder = LaneFinder(camera, perspective_transform, thresher)
for cur_fn in example_images:
img = cv2.imread(cur_fn)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
lane_finder.clear_history()
out_img, cam_img, persp = lane_finder.find_lanes_using_window(img)
fig = plt.figure(figsize=(12,8))
plt.title(cur_fn)
plt.imshow(cam_img)
plt.xlim(0, 1280)
plt.ylim(720, 0)
plt.show()
8. Creating top view videos¶
Here I verify how the lane detector reacts to a sequence of images and smoothes the variation between single frames
%run AdvLaneVideoCreator.py
create_top_video()
HTML("""
<video width="960" height="540" controls>
<source src="{0}">
</video>
""".format('test_videos_output/find_lanes_raw.mp4'))
create_top_video_photo()
HTML("""
<video width="960" height="540" controls>
<source src="{0}">
</video>
""".format('test_videos_output/find_lanes_raw_photo.mp4'))
9. Visualizing the detected lane lines in the original image¶
%run AdvLaneFinder.py
lane_finder = LaneFinder(camera, perspective_transform, thresher)
for cur_fn in example_images:
img = cv2.imread(cur_fn)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
lane_finder.clear_history()
out_img, cam_img, persp = lane_finder.find_lanes_using_window(img)
fig = plt.figure(figsize=(12,8))
plt.title(cur_fn)
plt.imshow(persp)
plt.xlim(0, 1280)
plt.ylim(720, 0)
plt.show()
10. Creation of the combined output video¶
%run AdvLaneVideoCreator.py
create_perspective_video()
HTML("""
<video width="960" height="540" controls>
<source src="{0}">
</video>
""".format('test_videos_output/combined.mp4'))