websocket傳輸canvas影象資料給C++服務端opencv影象實現web線上實時影象處理
阿新 • • 發佈：2019-03-30
前後端的耦合想了很久，上下課都在思考怎麼做，然後終於憋出來了。這是之前搞的一個視覺計算的專案，boss叫對接到前端，於是就產生了這樣一個詭異的需求，就是前端開啟攝像頭，同時需要把攝像頭的資料回傳到後端進行影象處理（比如美顏啊腦袋上加個裝飾品之類），這就需要涉及到前端和服務端的資料編碼耦合，想了想既然任何影象在記憶體裡面都是一個uchar矩陣，於是琢磨了這個東西出來。
首先假設我們的前端開啟websocket連線後端，連線上了以後前端開啟攝像頭取攝像頭資料傳輸給後端，後端通過一系列的影象處理機器學習以後編碼影象回傳給前端。
前端程式碼：
<html> <head> <title> Camera Test </title> </head> <body> <video style="display:none;" id="video" autoplay ></video> <canvas id="canvas" width="480" height="320"></canvas> <img id="target" width="480" height="320"></img> <h1>Action:Normal</h1> <script> var video = document.getElementById('video'); var canvas = document.getElementById('canvas'); var image = document.getElementById('target'); var capture = document.getElementById('capture'); var context = canvas.getContext('2d'); var ws = new WebSocket("ws://127.0.0.1:9002"); ws.binaryType = "arraybuffer"; ws.onopen = function() { ws.send("I'm client"); }; ws.onmessage = function (evt) { console.log("resive"); try{ //顯示後端回傳回來的base64影象 image.src="data:image/png;base64,"+evt.data; }catch{ } }; ws.onclose = function() { alert("Closed"); }; ws.onerror = function(err) { alert("Error: " + err); }; function getUserMediaToPhoto(constraints,success,error) { if(navigator.mediaDevices.getUserMedia){ //最新標準API navigator.mediaDevices.getUserMedia(constraints).then(success).catch(error); }else if (navigator.webkitGetUserMedia) { //webkit核心瀏覽器 navigator.webkitGetUserMedia(constraints,success,error); }else if(navigator.mozGetUserMedia){ //firefox瀏覽器 navigator.mozGetUserMedia(constraints,success,error); }else if(navigator.getUserMedia){ //舊版API navigator.getUserMedia(constraints,success,error); } } //成功回撥函式 function success(stream){ //相容webkit核心瀏覽器 var CompatibleURL = window.URL || window.webkitURL; //將視訊流轉化為video的源 video.src = CompatibleURL.createObjectURL(stream); //video.play();//播放視訊 } function error(error) { console.log('訪問使用者媒體失敗：',error.name,error.message); } if(navigator.mediaDevices.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia || navigator.getUserMedia){ getUserMediaToPhoto({video:{width:480,height:320}},success,error); }else{ alert('你的瀏覽器不支援訪問使用者媒體裝置'); } //這個函式是實現將canvas上面的base64影象轉為影象資料流的字串形式 function dataURItoBlob(dataURI) { // convert base64/URLEncoded data component to raw binary data held in a string var byteString; if (dataURI.split(',')[0].indexOf('base64') >= 0) byteString = atob(dataURI.split(',')[1]); else byteString = unescape(dataURI.split(',')[1]); // separate out the mime component var mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0]; // write the bytes of the string to a typed array var ia = new Uint8Array(byteString.length); for (var i = 0; i < byteString.length; i++) { ia[i] = byteString.charCodeAt(i); } return new Blob([ia], {type:mimeString}); } timer = setInterval( function () { context.drawImage(video,0,0,480,320); var data = canvas.toDataURL('image/jpeg', 1.0); newblob = dataURItoBlob(data); //將轉換好成為字串的影象資料傳送出去 ws.send(newblob); }, 50);//這裡我們的前端還是需要延時的，如果我們的後端計算實時性不是很強的話，而恰好我的專案後端計算規模非常大，所以需要50ms的等待 </script> </body> </html>
C++伺服器端（這裡需要使用到websocket++讀者請自行編譯）
network.h
#pragma once
namespace network::wsocket {
class sc_websocket{
public:
sc_websocket(int id, std::string address, int port) ;
void Run();
~sc_websocket();
private:
int port;
int id;
std::string address;
};
}
network.cpp
#include <websocketpp/config/asio_no_tls.hpp> #include <websocketpp/server.hpp> #include <websocketpp/config/asio_no_tls.hpp> #include <websocketpp/server.hpp> #include "network.h" using websocketpp::lib::placeholders::_1; using websocketpp::lib::placeholders::_2; using websocketpp::lib::bind; typedef websocketpp::server<websocketpp::config::asio> WebsocketServer; typedef WebsocketServer::message_ptr message_ptr; //解碼base64資料 static std::string base64Decode(const char* Data, int DataByte) { //解碼錶 const char DecodeTable[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, // '+' 0, 0, 0, 63, // '/' 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, // '0'-'9' 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, // 'A'-'Z' 0, 0, 0, 0, 0, 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, // 'a'-'z' }; std::string strDecode; int nValue; int i = 0; while (i < DataByte) { if (*Data != '\r' && *Data != '\n') { nValue = DecodeTable[*Data++] << 18; nValue += DecodeTable[*Data++] << 12; strDecode += (nValue & 0x00FF0000) >> 16; if (*Data != '=') { nValue += DecodeTable[*Data++] << 6; strDecode += (nValue & 0x0000FF00) >> 8; if (*Data != '=') { nValue += DecodeTable[*Data++]; strDecode += nValue & 0x000000FF; } } i += 4; } else { Data++; i++; } } return strDecode; } //編碼base64資料 static std::string base64Encode(const unsigned char* Data, int DataByte) { //編碼表 const char EncodeTable[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; //返回值 std::string strEncode; unsigned char Tmp[4] = { 0 }; int LineLength = 0; for (int i = 0; i < (int)(DataByte / 3); i++) { Tmp[1] = *Data++; Tmp[2] = *Data++; Tmp[3] = *Data++; strEncode += EncodeTable[Tmp[1] >> 2]; strEncode += EncodeTable[((Tmp[1] << 4) | (Tmp[2] >> 4)) & 0x3F]; strEncode += EncodeTable[((Tmp[2] << 2) | (Tmp[3] >> 6)) & 0x3F]; strEncode += EncodeTable[Tmp[3] & 0x3F]; if (LineLength += 4, LineLength == 76) { strEncode += "\r\n"; LineLength = 0; } } //對剩餘資料進行編碼 int Mod = DataByte % 3; if (Mod == 1) { Tmp[1] = *Data++; strEncode += EncodeTable[(Tmp[1] & 0xFC) >> 2]; strEncode += EncodeTable[((Tmp[1] & 0x03) << 4)]; strEncode += "=="; } else if (Mod == 2) { Tmp[1] = *Data++; Tmp[2] = *Data++; strEncode += EncodeTable[(Tmp[1] & 0xFC) >> 2]; strEncode += EncodeTable[((Tmp[1] & 0x03) << 4) | ((Tmp[2] & 0xF0) >> 4)]; strEncode += EncodeTable[((Tmp[2] & 0x0F) << 2)]; strEncode += "="; } return strEncode; } //imgType 包括png bmp jpg jpeg等opencv能夠進行編碼解碼的檔案 static std::string Mat2Base64(const cv::Mat &img, std::string imgType) { //Mat轉base64 std::string img_data; std::vector<uchar> vecImg; std::vector<int> vecCompression_params; vecCompression_params.push_back(CV_IMWRITE_JPEG_QUALITY); vecCompression_params.push_back(90); imgType = "." + imgType; //重點來了，它是負責把影象從opencv的Mat變成編碼好的影象位元流的重要函式 cv::imencode(imgType, img, vecImg, vecCompression_params); img_data = base64Encode(vecImg.data(), vecImg.size()); return img_data; } //base64轉Mat static cv::Mat Base2Mat(std::string &base64_data) { cv::Mat img; std::string s_mat; s_mat = base64Decode(base64_data.data(), base64_data.size()); std::vector<char> base64_img(s_mat.begin(), s_mat.end()); img = cv::imdecode(base64_img, CV_LOAD_IMAGE_COLOR); return img; } void OnOpen(WebsocketServer *server, websocketpp::connection_hdl hdl) { std::cout << "have client connected" << std::endl; } void OnClose(WebsocketServer *server, websocketpp::connection_hdl hdl) { std::cout << "have client disconnected" << std::endl; } void OnMessage(WebsocketServer *server, websocketpp::connection_hdl hdl, message_ptr msg) { std::string image_str = msg->get_payload(); std::vector<char>img_vec(image_str.begin(), image_str.end()); try { //把前端傳來的影象字串進行解碼 cv::Mat img = cv::imdecode(img_vec, CV_LOAD_IMAGE_COLOR); if (!img.empty()) { cv::imshow("", img); cv::Mat output = 你的影象處理函式(img); if (!output.empty()) { //把你處理完的影象轉換為字串返回給前端 std::string strRespon = Mat2Base64(output, "bmp"); server->send(hdl, strRespon, websocketpp::frame::opcode::text); } cv::waitKey(1); } } catch (const std::exception&) { std::cout <<" 解碼異常" << std::endl; } } namespace network::wsocket { sc_websocket::sc_websocket(int id, std::string address, int port) { } sc_websocket::~sc_websocket() { } void sc_websocket::Run() { WebsocketServer server; server.set_access_channels(websocketpp::log::alevel::all); server.clear_access_channels(websocketpp::log::alevel::frame_payload); // Initialize Asio server.init_asio(); // Register our message handler server.set_open_handler(bind(&OnOpen, &server, ::_1)); server.set_close_handler(bind(&OnClose, &server, _1)); server.set_message_handler(bind(&OnMessage, &server, _1, _2)); // Listen on port 9002 server.listen(9002); // Start the server accept loop server.start_accept(); // Start the ASIO io_service run loop server.run(); } }
程式碼難免在打字的時候打錯，有什麼問題聯絡筆者。整個服務端的實現難點無非在於編碼與解碼的方法保持客戶端和服務端資料耦合性，這個東西也琢磨了我好幾天才琢磨透，再接再厲把，io真的是一個神奇的東西，當你把它深刻的理解到記憶體的時候，它就像