From c54519de29aab1fbf82896659658ae58041d7127 Mon Sep 17 00:00:00 2001 From: zcy <290198252@qq.com> Date: Wed, 25 May 2022 21:34:18 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=B9=E6=88=90ssd=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E7=94=A8=E6=96=87=E4=BB=B6=E8=BE=93=E5=85=A5=E7=9A=84=E5=BD=A2?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../webrtc_capture/src/camera_video_sink.cpp | 2 + client/webrtc_capture/src/cv_ssd.cpp | 49 +++++-- client/webrtc_capture/src/cv_ssd.h | 3 +- client/webrtc_capture/src/mainwindow.cpp | 121 +++++++++++------- client/webrtc_capture/src/mainwindow.h | 2 + client/webrtc_capture/src/mainwindow.ui | 62 +++++++-- 6 files changed, 168 insertions(+), 71 deletions(-) diff --git a/client/webrtc_capture/src/camera_video_sink.cpp b/client/webrtc_capture/src/camera_video_sink.cpp index 179f666..f71f45d 100644 --- a/client/webrtc_capture/src/camera_video_sink.cpp +++ b/client/webrtc_capture/src/camera_video_sink.cpp @@ -93,9 +93,11 @@ void CameraVideoSink::OnFrame(const webrtc::VideoFrame& frame) { width*height/4); m_buf.PushFirst(data); + auto timestamp_curr = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()).count(); if(timestamp_curr - timestamp > 1000) { + qDebug()< -const size_t width = 1280; -const size_t height = 720; +#include + +using namespace std; + String labelFile = "ssd/labelmap_det.txt"; -String modelFile = "D:/project/multimedia/client/build-webrtc_capture-Desktop_Qt_5_15_2_MSVC2019_64bit-Debug/debug/ssd/MobileNetSSD_deploy.caffemodel"; -String model_text_file = "D:/project/multimedia/client/build-webrtc_capture-Desktop_Qt_5_15_2_MSVC2019_64bit-Debug/debug/ssd/MobileNetSSD_deploy.prototxt"; +String modelFile = "debug/ssd/MobileNetSSD_deploy.caffemodel"; +String model_text_file = "debug/ssd/MobileNetSSD_deploy.prototxt"; -String objNames[] = { "background", -"aeroplane", "bicycle", "bird", "boat", -"bottle", "bus", "car", "cat", "chair", -"cow", "diningtable", "dog", "horse", -"motorbike", "person", "pottedplant", -"sheep", "sofa", "train", "tvmonitor" }; +String objNames[] = { + "background", + "aeroplane", + "bicycle", + "bird", + "boat", + "bottle", + "bus", + "car", + "cat", + "chair", + "cow", + "diningtable", + "dog", + "horse", + "motorbike", + "person", + "pottedplant", + "sheep", + "sofa", + "train", + "tvmonitor" +}; -cv::Mat * ssd_detect(cv::Mat *inframe) { - if (inframe->empty()) { +cv::Mat * ssd_detect(cv::Mat *inframe,std::string modelFile,std::string model_text_file) { + if (inframe->empty() || (modelFile == "") || (model_text_file == "")) { printf("could not load image...\n"); return inframe; } @@ -28,7 +47,6 @@ cv::Mat * ssd_detect(cv::Mat *inframe) { Size(300, 300), Scalar(127.5, 127.5, 127.5), true, false); -// Net net = readNetFromCaffe(model_text_file, modelFile); //// image：这个就是我们将要输入神经网络进行处理或者分类的图片。 //// mean：需要将图片整体减去的平均值，如果我们需要对RGB图片的三个通道分别减去不同的值，那么可以使用3组平均值， @@ -40,6 +58,7 @@ cv::Mat * ssd_detect(cv::Mat *inframe) { //// size：这个参数是我们神经网络在训练的时候要求输入的图片尺寸。 //// swapRB：OpenCV中认为我们的图片通道顺序是BGR，但是我平均值假设的顺序是RGB， /// 所以如果需要交换R和G，那么就要使swapRB=true +/// Net net = readNetFromCaffe(model_text_file, modelFile); net.setInput(blobImage1, "data"); Mat detection = net.forward("detection_out"); @@ -56,9 +75,11 @@ cv::Mat * ssd_detect(cv::Mat *inframe) { Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y)); rectangle(*image2, object_box, Scalar(0, 0, 255), 2, 8, 0); - putText(*image2, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2); + putText(*image2, format("%s", objNames[objIndex].c_str()), + Point(tl_x, tl_y), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2); } } + cv::imshow("image2",*image2); return image2; } diff --git a/client/webrtc_capture/src/cv_ssd.h b/client/webrtc_capture/src/cv_ssd.h index 66f20db..5cdb93e 100644 --- a/client/webrtc_capture/src/cv_ssd.h +++ b/client/webrtc_capture/src/cv_ssd.h @@ -9,8 +9,7 @@ using namespace cv; using namespace cv::dnn; using namespace std; -cv::Mat * ssd_detect(cv::Mat*); - +cv::Mat * ssd_detect(cv::Mat *inframe,std::string modelFile,std::string model_text_file); diff --git a/client/webrtc_capture/src/mainwindow.cpp b/client/webrtc_capture/src/mainwindow.cpp index e432fbe..d4935fb 100644 --- a/client/webrtc_capture/src/mainwindow.cpp +++ b/client/webrtc_capture/src/mainwindow.cpp @@ -11,6 +11,7 @@ #include "cv_ssd.h" #include #include +#include class ASyncDetectAndRenderThread :public QSSASyncProcess{ @@ -20,19 +21,26 @@ public: STATUS_RUNNING = 2 }Status; ASyncDetectAndRenderThread(QWidget * parent,CPlayWidget *render_ui, - int width,int height){ + int width,int height, + std::string model_path, + std::string model_txt_path){ m_render = render_ui; m_parent = parent; m_status = STATUS_RUNNING; m_width = width; m_height = height; + this->m_model_path = model_path; + this->m_model_txt_path = model_txt_path; } - bool DetectStatus(){ + bool Detecting(){ if(m_mat.size() > 0) return true; else return false; } + Status DetectStatus(){ + return this->m_status; + } void SetCvImage(uint8_t *frame){ cv::Mat yuv420p; yuv420p.create(m_height*3/2, @@ -42,7 +50,6 @@ public: cv::Mat *rgbImg = new cv::Mat; cv::cvtColor(yuv420p, *rgbImg, cv::COLOR_YUV2BGR_I420); std::lock_guard guard(this->m_mutex); -// imshow("yuv420",yuv420p); m_mat.push_back(rgbImg); } @@ -50,12 +57,18 @@ public: while(m_status == STATUS_RUNNING){ cv::Mat *c = takeLast(); if(nullptr != c){ - cv::Mat *result = ssd_detect(c); + cv::Mat *result = ssd_detect(c, + this->m_model_path, + this->m_model_txt_path); + qDebug()<cols<rows<type(); cv::Mat yuvData; + cv::cvtColor(*result, yuvData, cv::COLOR_BGR2YUV_I420); this->m_render->OnCameraData(yuvData.data); + } + delete c; } } @@ -68,7 +81,7 @@ private: return ret; } return nullptr; - } + } std::list m_mat; std::mutex m_mutex; CPlayWidget *m_render; @@ -76,6 +89,8 @@ private: Status m_status; int m_width; int m_height; + std::string m_model_path; // 模型路径 + std::string m_model_txt_path; // 模型txt路径 }; ASyncDetectAndRenderThread *gdetect = nullptr; @@ -87,6 +102,7 @@ public: this->mfbs = p; mUI = render_ui; this->state = true; + } // 停止渲染 void StopRender(){ @@ -98,14 +114,14 @@ public: qtimer->setSingleShot(false); QObject::connect(qtimer, SIGNAL(timeout()), &eventLoop, SLOT(quit())); - qtimer->start(3); + qtimer->start(22); while(state){ if(mfbs ->Size() > 0){ uint8_t *frame = mfbs->TakeLast(); mUI->OnCameraData(frame); if(gdetect != nullptr){ - if(!gdetect->DetectStatus()){ + if(!gdetect->Detecting()){ gdetect->SetCvImage(frame); } } @@ -142,16 +158,8 @@ MainWindow::MainWindow(QWidget *parent) info->GetDeviceName(i,name,100,nullptr,0,nullptr,0); ui->comboBox->addItem(QString::asprintf("%s",name),i); } - origin_picture = new QLabel(this); - processed_picture = new QLabel(this); - - ui->gridLayout->addWidget(origin_picture,0,1); - origin_picture->setMaximumWidth(12000); - origin_picture->setMaximumHeight(12000); - - ui->gridLayout->addWidget(processed_picture,1,0); mDetectResut = new CPlayWidget(this); - ui->gridLayout->addWidget(mDetectResut,1,1); + ui->gridLayout->addWidget(mDetectResut,0,1); } @@ -167,37 +175,39 @@ void MainWindow::OnUpdateFrame( rtc::scoped_refptr& void MainWindow::on_pushButton_clicked() { + if(ui->pushButton->text() == QString("采集")) { + int id = ui->comboBox->currentData().toInt(); + webrtc::VideoCaptureCapability p; + std::unique_ptr info( + webrtc::VideoCaptureFactory::CreateDeviceInfo()); - int id = ui->comboBox->currentData().toInt(); - webrtc::VideoCaptureCapability p; + char ids[128]; + info->GetDeviceName(id,nullptr,0,ids,128,nullptr,0); + info->GetCapability(ids,id,p); + qDebug()< info( - webrtc::VideoCaptureFactory::CreateDeviceInfo()); + m_capturer.reset(CameraVideoSink::Create(p.width, p.height, 25, id)); + if (!m_capturer) { + qDebug()<<"error"; + } + ui->openGLWidget->SetDataType(CPlayWidget::TYPE_YUV420P); + ui->openGLWidget->SetImgSize(m_capturer->Capability().width, + m_capturer->Capability().height); + ui->openGLWidget->StartRender(); + // ui->openGLWidget->moveToThread(&gRender->Thread()); + if(gRender == nullptr){ + gRender = new AsyncRennder(this,m_capturer->VideoBuffer(),ui->openGLWidget); + gRender->Start(this); + connect(gRender,&QSSASyncProcess::Done,this,&MainWindow::RenderDone); + } + mDetectResut->SetImgSize(480,560); + mDetectResut->StartRender(); - char ids[128]; - info->GetDeviceName(id,nullptr,0,ids,128,nullptr,0); - info->GetCapability(ids,id,p); - qDebug()<pushButton->setText(QString::asprintf("正在采集")); + }else{ + ui->openGLWidget->StopRender(); } - ui->openGLWidget->SetDataType(CPlayWidget::TYPE_YUV420P); - ui->openGLWidget->SetImgSize(m_capturer->Capability().width, - m_capturer->Capability().height); - ui->openGLWidget->StartRender(); -// ui->openGLWidget->moveToThread(&gRender->Thread()); - if(gRender == nullptr){ - gRender = new AsyncRennder(this,m_capturer->VideoBuffer(),ui->openGLWidget); - gRender->Start(this); - connect(gRender,&QSSASyncProcess::Done,this,&MainWindow::RenderDone); - } - mDetectResut->SetImgSize(480,560); - mDetectResut->StartRender(); - - } void MainWindow::RenderDone() @@ -205,8 +215,6 @@ void MainWindow::RenderDone() } - - cv::Mat *QImage2cvMat(QImage image) { cv::Mat *mat; @@ -247,8 +255,13 @@ void MainWindow::on_pushButton_3_clicked() QImage image = origin_picture->pixmap()->toImage(); image = image.scaled(1280,720); cv::Mat pic = *QImage2cvMat(image); + if((ui->lineEdit->text() == "") || (ui->lineEdit_2->text() == "")){ - cv::Mat *result = ssd_detect(&pic); + } + cv::Mat *result = ssd_detect(&pic, + ui->lineEdit->text().toStdString(), + ui->lineEdit_2->text().toStdString() + ); this->processed_picture->setPixmap(QPixmap::fromImage(image)); } } @@ -258,10 +271,26 @@ void MainWindow::on_pushButton_4_clicked() { if(gdetect == nullptr){ gdetect = new ASyncDetectAndRenderThread(this,mDetectResut,ui->openGLWidget->RenderWidth(), - ui->openGLWidget->RenderHeight()); + ui->openGLWidget->RenderHeight(), + ui->lineEdit->text().toStdString(), + ui->lineEdit_2->text().toStdString()); gdetect->Start(this); connect(gdetect,&QSSASyncProcess::Done,this,&MainWindow::RenderDone); } } +// 导入文件 +void MainWindow::on_pushButton_5_clicked() +{ + QString label1 = ui->lineEdit_2->text(); + QString openFile = QFileDialog::getOpenFileName(this, "0.0", "", + "*.caffemodel *.prototxt",nullptr); + if (openFile.contains(".caffemodel")){ + ui->lineEdit->setText(openFile); + } + if (openFile.contains(".prototxt")){ + ui->lineEdit_2->setText(openFile); + } +} + diff --git a/client/webrtc_capture/src/mainwindow.h b/client/webrtc_capture/src/mainwindow.h index a8350f7..89a6d49 100644 --- a/client/webrtc_capture/src/mainwindow.h +++ b/client/webrtc_capture/src/mainwindow.h @@ -28,6 +28,8 @@ private slots: void on_pushButton_4_clicked(); + void on_pushButton_5_clicked(); + private: Ui::MainWindow *ui; std::unique_ptr m_capturer; diff --git a/client/webrtc_capture/src/mainwindow.ui b/client/webrtc_capture/src/mainwindow.ui index 5c231b8..86b6706 100644 --- a/client/webrtc_capture/src/mainwindow.ui +++ b/client/webrtc_capture/src/mainwindow.ui @@ -20,9 +20,9 @@ MainWindow - + - + @@ -70,13 +70,6 @@ - - - - 动态检测 - - - @@ -92,6 +85,57 @@ + + + + + + 模型文件: .caffemodel + + + + + + + + + + .prototxt: + + + + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + 导入 + + + + + + + 动态检测 + + + + +