Socket steaming

Intel realsense

 

data sheet : https://www.intelrealsense.com/wp-content/uploads/2022/10/Intel-RealSense-D457-Datasheet-September-2022.pdf

 

 

Realsense 카메라를 통한 실시간 양방향 스트리밍 

import os
import socket
import cv2
import numpy as np
import base64
import glob
import sys
import time
import threading

class ServerSocket:

    def __init__(self, ip, port):
        self.TCP_IP = ip
        self.TCP_PORT = port
        
        self.socketOpen()
        self.receiveThread = threading.Thread(target=self.receiveImages)
        self.receiveThread.start()
    
    def socketClose(self):
        self.sock.close()
        print(u'Server socket [ TCP_IP: ' + self.TCP_IP + ', TCP_PORT: ' + str(self.TCP_PORT) + ' ] is close')
    
    def socketOpen(self):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.bind(self.TCP_IP, self.TCP_PORT)
        self.sock.listen(1)
        print(u'Server socket [ TCP_IP: ' + self.TCP_IP + ', TCP_PORT: ' + str(self.TCP_PORT) + ' ] is open')
        self.conn, self.addr = self.sock.accept()
        print(u'Server socket [ TCP_IP: ' + self.TCP_IP + ', TCP_PORT: ' + str(self.TCP_PORT) + ' ] is connected with client')

    def receiveImages(self):
        cnt_str = ''
        cnt =0

        try:
            while True:
                length = self.recvall(self.conn, 64)
                length1 = length.decode('utf-8')
                stringdat = self.recvall(self.conn, int(length1))
                stringdat2 = self.recvall(self.conn, int(length))
                stime = self.recvall(self.conn, 64)
                print('send time ', stime+decode('utf-8'))
                now = time.localtime()
                print('receive time: ' + datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f'))

                data = np.frombuffer(base64.b64decode(stringdat),np.uint8)
                decimg = cv2.imdecode(data,1)

                cv2.imshow('img',decimg)
                cv2.waitKey(1)

        except Exception as e:
            print(e)
            self.socketClose()
            cv2.destroyAllWindows()
            self.socketOpen()
            self.receiveThread = threading.Thread(target=self.receiveImages)
            self.receiveThread.start()
    
    def recvall(self, sock, count):
        buf = b''
        while count:
            newbuf = sock.recv(count)
            if not newbuf: return None
            buf += newbuf
            count -= len(newbuf)
        return buf
    

def main():
    server = ServerSocket('localhost', 8080)

if __name__ == "__main__":
    main()

Server code

 

서버측에서 헤더 수신 및 관련 인코딩 수치를 받아 사전 확인 및 동시 스트리밍

버퍼에 스트리밍 값 순차 입력 및 출력

 

Client code

import socket
import cv2
import numpy as np 
import time
import base64
import sys
from datetime import datetime
import pyrealsense2 as rs

sys.setrecursionlimit(5000)

class ClientVideoSocket:
    
    def __init__(self, ip, port):
        self.TCP_SERVER_IP = ip
        self.TCP_SERVER_PORT = port
        self.pipline = rs.pipeline()
        self.connectCount = 0
        self.connectServer()
        self.config = rs.config()
        self.config.enable_stream(rs.stream.depth, 640,480, rs.format.z16, 30)
        self.config.enable_stream(rs.stream.color, 640,480, rs.format.bgr8, 30)
        self.pipline.start(config)
    
    def connectServer(self):
        try:
            self.sock = socket.socket()
            self.sock.connect((self.TCP_SERVER_IP, self.TCP_SERVER_PORT))
            print(u'Client socket is connected with Server socket [ TCP_SERVER_IP: ' + self.TCP_SERVER_IP + ', TCP_SERVER_PORT: ' + str(self.TCP_SERVER_PORT) + ' ]')
            self.connectCount = 0
            self.sendImages()
        except Exception as e:
            print(e)
            self.connectCount += 1
            #create endurance sequence
            if self.connectCount == 10:
                print(u'Connect fail %d times. exit program'%(self.connectCount))
                sys.exit()
            print(u'%d times try to connect with server'%(self.connectCount))
            time.sleep(1)
            self.connectServer()
        
    def sendImages(self):
        cnt = 0
        pipeline = rs.pipeline() #센서연결
        config = rs.config() # 설정
        config.enable_stream(rs.stream.depth, 640, 480, rs.format.z16, 30) # 설정변경
        config.enable_stream(rs.stream.color, 640, 480, rs.format.bgr8, 30) #설정변경
        pipeline.start(config)
        try:
            while True:
                frames = pipeline.wait_for_frames()
                depth_frame = frames.get_depth_frame()
                color_frame = frames.get_color_frame()

                if not depth_frame or not color_frame:
                    continue
                
                depth_image = np.asanyarray(depth_frame.get_data()) 
                color_image = np.asanyarray(color_frame.get_data())
                depth_colormap = cv2.applyColorMap(cv2.convertScaleAbs(depth_image, alpha=0.03), cv2.COLORMAP_JET)

                stime = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')

                encode_param=[int(cv2.IMWRITE_JPEG_QUALITY),90]
                result, imgencode = cv2.imencode('.jpg', depth_colormap, encode_param)
                result2, imgencode2 = cv2.imencode('.jpg', color_image, encode_param)
                data = np.array(imgencode)
                data2 = np.array(imgencode2)
                stringData1 = base64.b64encode(data)
                stringData2 = base64.b64encode(data2)
                length = str(len(stringData1))
                length2 = str(len(stringData2))
                send_length = length+','+length2
                self.sock.sendall(send_length.encode('utf-8').ljust(64))
                self.sock.send(stringData1)
                self.sock.send(stringData2)
                self.sock.send(stime.encode('utf-8').ljust(64))
                print(u'send images %d'%(cnt))
                cnt+=1
                time.sleep(0.02)
        except Exception as e:
            print(e)
            self.sock.close()
            time.sleep(1)
            self.connectServer()
            self.sendImages()

def main():
    TCP_IP = 'localhost'
    TCP_PORT = 8080 
    client = ClientVideoSocket(TCP_IP, TCP_PORT)

if __name__=="__main__":
    main()

Lidar sensor 값과 color 값 동시에 전송 및 수신 측에서 동시 재생