使用 dnn 对象检测滞后分析 android 中的 opencv 图像

问题描述

我已经完成了这个 [1]https://docs.opencv.org/3.4/d0/d6c/tutorial_dnn_android.html 教程，您可以使用手机的摄像头和 opencv 在 android 上运行深度神经网络 (dnn)。使用三星 galaxy S9+，我没有明显的延迟，看起来是实时的。我在 python 中实现了一个服务器脚本，用于 raspBerry pi 4 以使用它的 PI 摄像头将帧发送到手机，由 dnn 而不是手机的摄像头进行分析。我在手机中单独发送和显示帧时没有延迟，但是当我通过 dnn 运行它时，我得到了大约 5 秒的延迟。使用相同的 dnn 代码，我应该不会有任何延迟，因为它已经与手机的相机配合使用了。我试过给它自己的线程并改变图像的编码方式，但没有任何变化。我认为相机教程之间唯一改变的是套接字以及图像的编码方式，但套接字端没有延迟。下面是我在 python 中的服务器端代码和来自 android 的主要代码片段.

代码说明：pi 4 中的 server.py 从视频源发送帧，首先发送帧的大小，然后发送帧本身。在android中代码的onCreate中，加载dnn网络然后运行'connect'线程连接到服务器。在那里，建立了连接。它首先获取正在发送的数据包（帧）的大小，然后获取帧本身。然后将其转换回 Mat 类型，通过 dnn，并转换为位图以在手机中显示。

Server.py

import socket,cv2,pickle,struct,imutils,pickle

# import thread module 
from threading import Thread

# Socket Create
server_socket = socket.socket(socket.AF_INET,socket.soCK_STREAM)

host_ip = ''
port = 1111

vid = cv2.VideoCapture(0)

#threaded connections
def connect(c,v):

    #encode_param = [int(cv2.IMWRITE_PNG_STRATEGY_RLE),90]
        
    with c:
        while True:
            #read frame
            img,frame = v.read()
            
            #resize and encode to jpg
            frame = imutils.resize(frame,width=320)
            result,frame = cv2.imencode('.jpg',frame)

            #convert to bytes and get size
            data = bytes(frame)    
            size = len(data)
            
            #send size of frame in big indian byte order
            c.sendall(size.to_bytes(4,byteorder='big'))
            
            #send frame
            c.sendall(frame)

            print("sent: " + str(size) + " bytes")


socket_address = (host_ip,port)

# Socket Bind
server_socket.bind(socket_address)

# Socket Listen
server_socket.listen(5)
print("LISTENING AT:",socket_address)

# Socket Accept
while True:
    client_socket,addr = server_socket.accept()
    print('GOT CONNECTION FROM:',addr)
    
    #start connection thread
    new_thread = Thread(target = connect,args = (client_socket,vid))
    new_thread.start()
        
server_socket.close()
cam.release()

Android Java 代码段

@Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);

        //launch opencv manager
        OpenCVLoader.initAsync(OpenCVLoader.OPENCV_VERSION,this,mloaderCallback);

        setContentView(R.layout.activity_main);

        //setup image and text view
        imagev = (ImageView) findViewById(R.id.opencvImageView);
        networkstatus = (TextView) findViewById(R.id.statusText);

        //create ddn network
        createDDNNetwork();

        //run socket
        connect.start();

    }

//Load a network.
    public void createDDNNetwork() {
        String proto = getPath("MobileNetSSD_deploy.prototxt",this);
        String weights = getPath("MobileNetSSD_deploy.caffemodel",this);
        net = Dnn.readNetFromCaffe(proto,weights);
        Log.i(TAG,"Network loaded successfully");
        //networkstatus.setText("DDN Network loaded successfully");
    }

    //identify objects in frame
    public Mat identify(Mat frame) {
        final int IN_WIDTH = 300;
        final int IN_HEIGHT = 300;
        final float WH_RATIO = (float)IN_WIDTH / IN_HEIGHT;
        final double IN_SCALE_FACTOR = 0.007843;
        final double MEAN_VAL = 127.5;
        final double THRESHOLD = 0.7;

        // Get a new frame
       // Mat frame = inputFrame;
        //Imgproc.cvtColor(frame,frame,Imgproc.COLOR_RGBA2RGB);

        // Forward image through network.
        Mat blob = Dnn.blobFromImage(frame,IN_SCALE_FACTOR,new Size(IN_WIDTH,IN_HEIGHT),new Scalar(MEAN_VAL,MEAN_VAL,MEAN_VAL),/*swapRB*/false,/*croP*/false);

        net.setInput(blob);
        Mat detections = net.forward();

        int cols = frame.cols();
        int rows = frame.rows();

        detections = detections.reshape(1,(int)detections.total() / 7);

        for (int i = 0; i < detections.rows(); ++i) {

            double confidence = detections.get(i,2)[0];
            int classId = (int)detections.get(i,1)[0];

            if (confidence > THRESHOLD && classId == 15) {


                int left   = (int)(detections.get(i,3)[0] * cols);
                int top    = (int)(detections.get(i,4)[0] * rows);
                int right  = (int)(detections.get(i,5)[0] * cols);
                int bottom = (int)(detections.get(i,6)[0] * rows);

                // Draw rectangle around detected object.
                Imgproc.rectangle(frame,new Point(left,top),new Point(right,bottom),new Scalar(0,255,0));

                String label = classNames[classId] + ": " + confidence;
                int[] baseLine = new int[1];
                Size labelSize = Imgproc.getTextSize(label,Core.FONT_HERShey_SIMPLEX,0.5,1,baseLine);

                // Draw background for label.
                Imgproc.rectangle(frame,top - labelSize.height),new Point(left + labelSize.width,top + baseLine[0]),new Scalar(255,255));

                // Write class name and confidence.
                Imgproc.putText(frame,label,0));
            }
        }
        return frame;
    }

//connect to server
    Thread connect = new Thread(new Runnable() {

        @Override
        public void run() {
            // establish a connection
            try
            {
                socket = new Socket(IP,PORT);
                System.out.println("Connected");
                networkstatus.setText("Connected!");
                input   = new DataInputStream(socket.getInputStream());

                // String mssg = reply.readUTF();
                // System.out.println("server reply: " + mssg);
            }
            catch(UnkNownHostException u)
            {
                System.out.println(u);
            }
            catch(IOException i)
            {
                System.out.println(i);
            }

            String data = "";
            myFrame = new Mat();

            byte[] fr;
            int bytesToRead;
            int bytesRead;
            Mat fanalyze;
            byte[] size_buff;
            //ByteBuffer byteBuffer;

            // keep reading until "Over" is input
            while (!data.equals("over"))
            {
                try
                {

                    // Get size of packet
                    size_buff = new byte[4];
                    input.read(size_buff);
                    bytesToRead = ByteBuffer.wrap(size_buff).asIntBuffer().get();

                    //byte frame
                    fr = new byte[bytesToRead];
                    //bytesRead = 0;

                    //get frame
                    input.readFully(fr);

                    //convert binary to MAT
                    myFrame =  Imgcodecs.imdecode(new MatOfByte(fr),Imgcodecs.IMREAD_COLOR);

                    //show packet info
                    String info = ",w:" + myFrame.width() + ",h: " + myFrame.height();
                    System.out.println("Packet: " + fr + ",packet size: " + bytesToRead + info);

                    myFrame = identify(myFrame);

                    //update image view
                    runOnUiThread(new Runnable() {

                        @Override
                        public void run() {

                            //convert to bitmap
                            bitmap = convertMatToBitMap(myFrame);

                            imagev.setimageBitmap(bitmap);

                        }
                    });

                }
                catch(IOException i)
                {
                    System.out.println(i);
                }
            }

            // close the connection
            try
            {
                // out.writeUTF("Goodbye!");
                input.close();
                // out.close();
                socket.close();
                System.out.println("successfully closed");
            }
            catch(IOException i)
            {
                System.out.println(i);
            }

        }
    });

    //convert mat to bitmap
    private static Bitmap convertMatToBitMap(Mat input){
        Bitmap bmp = null;
        Mat rgb = new Mat();
        Imgproc.cvtColor(input,rgb,Imgproc.COLOR_BGR2RGB);

        try {
            bmp = Bitmap.createBitmap(rgb.cols(),rgb.rows(),Bitmap.Config.ARGB_8888);
            Utils.matToBitmap(rgb,bmp);
        }
        catch (CvException e){
            Log.d("Exception",e.getMessage());
        }
        return bmp;
    }


  [1]: https://docs.opencv.org/3.4/d0/d6c/tutorial_dnn_android.html

解决方法

暂无找到可以解决该程序问题的有效方法，小编努力寻找整理中！

如果你已经找到好的解决方法，欢迎将解决方案带上本链接一起发送给小编。

小编邮箱:dio#foxmail.com (将#修改为@）