Intel RealSense 를 활용한 YOLOv8 RealTime Object Segmentation

 

 

지난 글에서는 YOLOv8 을 이용하여

RGB camera input을 Realtime Object Segmentation 수행하였습니다.

YOLOv8 으로 RealTime Object Segmentation 수행하기

 

이제 어떤 물체를 segmentation 하는 것과 동시에

전체 pixel에 대한 평균 Depth를 계산하여 거리 정보를 추출하도록 만들어보겠습니다.

 

저는 Intel RealSense D435 Depth Camera를 이용하였고,

Object Segmentation 된 픽셀 부분들을 Depth Stream으로부터 정보를 얻어와

평균을 계산하는 방식입니다.

 

전체 코드입니다.

	import pyrealsense2 as rs
	import numpy as np
	import cv2
	import random
	from ultralytics import YOLO
	model = YOLO("yolov8n-seg.pt") # Load Model
	conf = 0.5 # confidence
	# Assign random color to each class
	yolo_classes = list(model.names.values())
	classes_ids = [yolo_classes.index(clas) for clas in yolo_classes]
	colors = {cls_id: random.choices(range(256), k=3) for cls_id in classes_ids}
	# Create a config and configure the pipeline to stream for Realsense
	pipeline = rs.pipeline()
	config = rs.config()
	# Get device product line for setting a supporting resolution
	pipeline_wrapper = rs.pipeline_wrapper(pipeline)
	pipeline_profile = config.resolve(pipeline_wrapper)
	device = pipeline_profile.get_device()
	device_product_line = str(device.get_info(rs.camera_info.product_line))
	found_rgb = False
	for s in device.sensors:
	if s.get_info(rs.camera_info.name) == 'RGB Camera':
	found_rgb = True
	break
	if not found_rgb:
	print("RGB Channel Not Found!")
	exit(0)
	config.enable_stream(rs.stream.depth, 640, 480, rs.format.z16, 30)
	# 화면 사이즈 변경을 원하면 여기를 수정
	config.enable_stream(rs.stream.color, 960, 540, rs.format.bgr8, 30)
	# Start streaming
	profile = pipeline.start(config)
	# Create an align object
	# rs.align allows us to perform alignment of depth frames to others frames
	# The "align_to" is the stream type to which we plan to align depth frames.
	align_to = rs.stream.color
	align = rs.align(align_to)
	try:
	while True:
	# Get frameset of color and depth
	frames = pipeline.wait_for_frames()
	# Align the depth frame to color frame
	aligned_frames = align.process(frames)
	# Get aligned frames
	depth_frame = aligned_frames.get_depth_frame() # aligned_depth_frame is a 640x480 depth image
	color_frame = aligned_frames.get_color_frame()
	# Validate that both frames are valid
	if not depth_frame or not color_frame:
	continue
	depth_image = np.asanyarray(depth_frame.get_data())
	color_image = np.asanyarray(color_frame.get_data())
	frame = color_image.copy()
	# Depth Image scailing 및 색체 조절
	depth_colormap = cv2.applyColorMap(cv2.convertScaleAbs(depth_image, alpha=0.075), cv2.COLORMAP_JET)
	# yolov8 처리
	results = model.predict(color_image, conf=conf)
	# Rendering
	for result in results:
	# Case : Nothing detected
	if result.masks is None or result.boxes is None:
	continue
	for mask, box in zip(result.masks.xy, result.boxes):
	points = np.int32([mask])
	color_number = int(box.cls[0])
	overlay = frame.copy() # 투명도 적용을 위해 복사본 생성
	cv2.fillPoly(overlay, points, colors[color_number]) # 복사본에 mask 적용
	alpha = 0.4 # 투명도 (0: 완전 투명, 1: 완전 불투명)
	frame = cv2.addWeighted(overlay, alpha, frame, 1 - alpha, 0) # 복사본과 원본 결합
	# Calculate and print the average depth for the mask
	mask_poly = np.zeros_like(depth_image, dtype=np.uint8)
	cv2.fillPoly(mask_poly, points, 255) # Create a mask polygon on the depth image
	masked_depth = cv2.bitwise_and(depth_image, depth_image, mask=mask_poly)
	average_depth = np.sum(masked_depth) / np.count_nonzero(masked_depth)
	#print(f"Average depth for class {model.names[color_number]}: {average_depth:.2f}mm")
	# Draw bounding box and depth text on the frame
	x, y, w, h = cv2.boundingRect(points)
	cv2.rectangle(frame, (x, y), (x + w, y + h), colors[color_number], 2)
	depth_text = f"Depth: {average_depth:.1f}mm"
	cv2.putText(frame, depth_text, (x, y - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, colors[color_number], 2)
	# Result
	#cv2.imshow("RGB Stream", color_image)
	#cv2.imshow("Depth Stream", depth_colormap)
	cv2.imshow("YOLOv8 Segmentation", frame)
	# Press esc or 'q' to close the image window
	key = cv2.waitKey(1)
	if key & 0xFF == ord('q') or key == 27:
	cv2.destroyAllWindows()
	break
	finally:
	pipeline.stop()

view raw yolov8_seg_realsense.py hosted with ❤ by GitHub

 

실행 결과입니다.

 

실제 거리와 대조를 해보았는데, 잘 일치하는 것을 알 수 있었습니다.

다만 튀는 값을 포함하여 전체 픽셀에 대해 평균을 매기니,

다소 굴곡이 많은 물체에 대해서는 수정이 필요할 듯 합니다.