I am currently working on a project that detects specific objects in a video stream. If a particular object is detected, an alarm of some sort will be rung. An HTML page will display the object detection output and the name of the object detected. I have used flask with python for the HTML page. The video from the ESP32 Camera is given to a python script through a URL; then, the output will be displayed on the HTML page. I am not using the camera webserver rather using a flask web app. I have completed the object detection part and the object detection output successfully. But I can't seem to figure out a way to control the alarm system attached to the ESP32-Cam through the python script. This is my python code
from pickletools import read_uint1
from flask import Flask, render_template, Response
import cv2
import cvzone
#url = 'C:/Users/Mukesh/Downloads/videoplayback.mp4'
url = '192.168.35.7:81/stream'
classNames = []
classFile = 'C:/Users/Mukesh/Desktop/Mini Project/Code/AIM/coco.names'
with open(classFile, 'rt') as f:
classNames = f.read().split('\n')
configPath = 'C:/Users/Mukesh/Desktop/Mini Project/Code/AIM/ssd_mobilenet_v3_large_coco_2020_01_14.pbtxt'
weightsPath = "C:/Users/Mukesh/Desktop/Mini Project/Code/AIM/frozen_inference_graph.pb"
net = cv2.dnn_DetectionModel(weightsPath, configPath)
net.setInputSize(320, 320)
net.setInputScale(1.0 / 127.5)
net.setInputMean((127.5, 127.5, 127.5))
net.setInputSwapRB(True)
app = Flask(name)
thres = 0.55
nmsThres = 0.2
cap = cv2.VideoCapture(url)
cap.set(3, 640)
cap.set(4, 480)
def gen_frames():
while 1:
isTrue, img = cap.read()
img = cv2.flip(img, 1)
if img is not None:
classIds, confs, bbox = net.detect(
img, confThreshold=thres, nmsThreshold=nmsThres)
try:
for classId, conf, box in zip(classIds.flatten(), confs.flatten(), bbox):
id = classNames[classId - 1]
#Send ID to HTML page
print(id)
if id=='drone':
cvzone.cornerRect(img, box)
cv2.putText(img, f'{classNames[classId - 1].upper()} {round(conf * 100, 2)}',
(box[0] + 10, box[1] +
30), cv2.FONT_HERSHEY_COMPLEX_SMALL,
1, (0, 255, 0), 2)
frame = img
ret, buffer = cv2.imencode('.jpg', frame)
frame = buffer.tobytes()
cv2.imshow('',img)
if cv2.waitKey(20)&0xff==ord(' '):
break
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')
except:
pass
cv2.waitKey(1)
else:
break
@app.route('/video_feed')
def video_feed():
return Response(gen_frames(), mimetype='multipart/x-mixed-replace; boundary=frame')
@app.route('/')
def home():
"""Video streaming home page."""
return render_template('Home.html')
if name == 'main':
app.run(debug=True)
This is the webpage
<!DOCTYPE html>
<html lang="en">
<head>
<title>Home</title>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<link href="https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css" rel="stylesheet">
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css"
integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js" type="text/javascript"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/js/bootstrap.bundle.min.js"></script>
<style>
body {
font-family: 'Gill Sans', 'Gill Sans MT', Calibri, 'Trebuchet MS', sans-serif;
background-image: linear-gradient(to right, #2E3192, #1BFFFF);
background-position: center;
background-repeat: no-repeat;
background-size: cover;
position: relative;
min-height: 100vh;
display: flex;
flex-flow: row;
}
.booth {
flex: 1 1 auto;
}
.f2 {
flex: 1 1 auto;
display: flex;
flex-flow: column-reverse;
}
.f2 .text {
flex: 0 1 auto;
}
.f2 .frame {
flex: 1 1 auto;
position: relative;
display: flex;
flex-flow: column;
}
.f2 .frame #click-photo {
flex: 0 1 auto;
}
.f2 .frame #canvas {
flex: 1 1 auto;
}
#canvas {
background-color: red;
}
</style>
</head>
<body>
<div class="booth">
<!--video id="video" width="100%" height="100%" autoplay></video-->
<img id="video" src="{{ url_for('video_feed') }}" width="100%">
</div>
<div class="f2">
<div class="frame">
<canvas id="canvas"></canvas>
<button id="click-photo">Click Photo</button>
</div>
<div class="text">
<h2 id="status" src="{{ url_for('obj')}}"></h2>
<!---Drone status--->
<h2 id="cam"></h2>
<!--Camera location-->
</div>
</div>
<script>
let video = document.querySelector("#video");
let click_button = document.querySelector("#click-photo");
let canvas = document.querySelector("#canvas");
click_button.addEventListener('click', function () {
canvas.getContext('2d').drawImage(video, 0, 0, canvas.width, canvas.height);
let image_data_url = canvas.toDataURL('image/jpeg');
console.log(image_data_url);
//document.getElementById("status").innerHTML = " Drone: detected";
document.getElementById("cam").innerHTML = "Camera: 1";
});
var stop = function () {
var stream = video.srcObject;
var tracks = stream.getTracks();
for (var i = 0; i < tracks.length; i++) {
var track = tracks[i];
track.stop();
}
video.srcObject = null;
}
var start = function () {
var video = document.getElementById('video'),
vendorUrl = window.URL || window.webkitURL;
if (navigator.mediaDevices.getUserMedia) {
navigator.mediaDevices.getUserMedia({ video: true })
.then(function (stream) {
video.srcObject = stream;
}).catch(function (error) {
console.log("Something went wrong!");
});
}
}
$(function () {
start();
});
</script>
</body>
</html>
This is the ESP32 Camera code
#include "esp_camera.h"
#include <WiFi.h>
#include "esp_timer.h"
#include "img_converters.h"
#include "Arduino.h"
#include "fb_gfx.h"
#include "soc/soc.h" //disable brownout problems
#include "soc/rtc_cntl_reg.h" //disable brownout problems
#include "esp_http_server.h"
#include <ESP32Servo.h>
//Replace with your network credentials
const char* ssid = "Mukesh";
const char* password = "qwertyuiop";
#define PART_BOUNDARY "123456789000000000000987654321"
// This project was tested with the AI Thinker Model, M5STACK PSRAM Model and M5STACK WITHOUT PSRAM
#define CAMERA_MODEL_AI_THINKER
//#define CAMERA_MODEL_M5STACK_PSRAM
//#define CAMERA_MODEL_M5STACK_WITHOUT_PSRAM
// Not tested with this model
//#define CAMERA_MODEL_WROVER_KIT
#if defined(CAMERA_MODEL_WROVER_KIT)
#define PWDN_GPIO_NUM -1
#define RESET_GPIO_NUM -1
#define XCLK_GPIO_NUM 21
#define SIOD_GPIO_NUM 26
#define SIOC_GPIO_NUM 27
#define Y9_GPIO_NUM 35
#define Y8_GPIO_NUM 34
#define Y7_GPIO_NUM 39
#define Y6_GPIO_NUM 36
#define Y5_GPIO_NUM 19
#define Y4_GPIO_NUM 18
#define Y3_GPIO_NUM 5
#define Y2_GPIO_NUM 4
#define VSYNC_GPIO_NUM 25
#define HREF_GPIO_NUM 23
#define PCLK_GPIO_NUM 22
#elif defined(CAMERA_MODEL_M5STACK_PSRAM)
#define PWDN_GPIO_NUM -1
#define RESET_GPIO_NUM 15
#define XCLK_GPIO_NUM 27
#define SIOD_GPIO_NUM 25
#define SIOC_GPIO_NUM 23
#define Y9_GPIO_NUM 19
#define Y8_GPIO_NUM 36
#define Y7_GPIO_NUM 18
#define Y6_GPIO_NUM 39
#define Y5_GPIO_NUM 5
#define Y4_GPIO_NUM 34
#define Y3_GPIO_NUM 35
#define Y2_GPIO_NUM 32
#define VSYNC_GPIO_NUM 22
#define HREF_GPIO_NUM 26
#define PCLK_GPIO_NUM 21
#elif defined(CAMERA_MODEL_M5STACK_WITHOUT_PSRAM)
#define PWDN_GPIO_NUM -1
#define RESET_GPIO_NUM 15
#define XCLK_GPIO_NUM 27
#define SIOD_GPIO_NUM 25
#define SIOC_GPIO_NUM 23
#define Y9_GPIO_NUM 19
#define Y8_GPIO_NUM 36
#define Y7_GPIO_NUM 18
#define Y6_GPIO_NUM 39
#define Y5_GPIO_NUM 5
#define Y4_GPIO_NUM 34
#define Y3_GPIO_NUM 35
#define Y2_GPIO_NUM 17
#define VSYNC_GPIO_NUM 22
#define HREF_GPIO_NUM 26
#define PCLK_GPIO_NUM 21
#elif defined(CAMERA_MODEL_AI_THINKER)
#define PWDN_GPIO_NUM 32
#define RESET_GPIO_NUM -1
#define XCLK_GPIO_NUM 0
#define SIOD_GPIO_NUM 26
#define SIOC_GPIO_NUM 27
#define Y9_GPIO_NUM 35
#define Y8_GPIO_NUM 34
#define Y7_GPIO_NUM 39
#define Y6_GPIO_NUM 36
#define Y5_GPIO_NUM 21
#define Y4_GPIO_NUM 19
#define Y3_GPIO_NUM 18
#define Y2_GPIO_NUM 5
#define VSYNC_GPIO_NUM 25
#define HREF_GPIO_NUM 23
#define PCLK_GPIO_NUM 22
#else
#error "Camera model not selected"
#endif
// #define Ser_1 14
// #define Ser_2 15
// #define ser_step 5
// Servo sn1;
// Servo sn2;
// Servo s1;
// Servo s2;
// int pos1=0;
// int pos2=0;
static const char* _STREAM_CONTENT_TYPE = "multipart/x-mixed-replace;boundary=" PART_BOUNDARY;
static const char* _STREAM_BOUNDARY = "\r\n--" PART_BOUNDARY "\r\n";
static const char* _STREAM_PART = "Content-Type: image/jpeg\r\nContent-Length: %u\r\n\r\n";
httpd_handle_t stream_httpd = NULL;
static esp_err_t stream_handler(httpd_req_t req){
camera_fb_t fb = NULL;
esp_err_t res = ESP_OK;
size_t _jpg_buf_len = 0;
uint8_t * _jpg_buf = NULL;
char * part_buf[64];
res = httpd_resp_set_type(req, _STREAM_CONTENT_TYPE);
if(res != ESP_OK){
return res;
}
while(true){
fb = esp_camera_fb_get();
if (!fb) {
Serial.println("Camera capture failed");
res = ESP_FAIL;
} else {
if(fb->width > 400){
if(fb->format != PIXFORMAT_JPEG){
bool jpeg_converted = frame2jpg(fb, 80, &_jpg_buf, &_jpg_buf_len);
esp_camera_fb_return(fb);
fb = NULL;
if(!jpeg_converted){
Serial.println("JPEG compression failed");
res = ESP_FAIL;
}
} else {
_jpg_buf_len = fb->len;
_jpg_buf = fb->buf;
}
}
}
if(res == ESP_OK){
size_t hlen = snprintf((char )part_buf, 64, _STREAM_PART, _jpg_buf_len);
res = httpd_resp_send_chunk(req, (const char )part_buf, hlen);
}
if(res == ESP_OK){
res = httpd_resp_send_chunk(req, (const char *)_jpg_buf, _jpg_buf_len);
}
if(res == ESP_OK){
res = httpd_resp_send_chunk(req, _STREAM_BOUNDARY, strlen(_STREAM_BOUNDARY));
}
if(fb){
esp_camera_fb_return(fb);
fb = NULL;
_jpg_buf = NULL;
} else if(_jpg_buf){
free(_jpg_buf);
_jpg_buf = NULL;
}
if(res != ESP_OK){
break;
}
//Serial.printf("MJPG: %uB\n",(uint32_t)(_jpg_buf_len));
}
return res;
}
void startCameraServer(){
httpd_config_t config = HTTPD_DEFAULT_CONFIG();
config.server_port = 80;
httpd_uri_t index_uri = {
.uri = "/",
.method = HTTP_GET,
.handler = stream_handler,
.user_ctx = NULL
};
//Serial.printf("Starting web server on port: '%d'\n", config.server_port);
if (httpd_start(&stream_httpd, &config) == ESP_OK) {
httpd_register_uri_handler(stream_httpd, &index_uri);
}
}
void setup() {
WRITE_PERI_REG(RTC_CNTL_BROWN_OUT_REG, 0); //disable brownout detector
// s1.setPeriodHertz(50);
// s2.setPeriodHertz(50);
// sn1.attach(2,1000,2000);
// sn2.attach(13,1000,2000);
// s1.attach(Ser_1,1000,2000);
// s2.attach(Ser_2,1000,2000);
// s1.write(pos1);
// s2.write(pos2);
Serial.begin(115200);
Serial.setDebugOutput(false);
camera_config_t config;
config.ledc_channel = LEDC_CHANNEL_0;
config.ledc_timer = LEDC_TIMER_0;
config.pin_d0 = Y2_GPIO_NUM;
config.pin_d1 = Y3_GPIO_NUM;
config.pin_d2 = Y4_GPIO_NUM;
config.pin_d3 = Y5_GPIO_NUM;
config.pin_d4 = Y6_GPIO_NUM;
config.pin_d5 = Y7_GPIO_NUM;
config.pin_d6 = Y8_GPIO_NUM;
config.pin_d7 = Y9_GPIO_NUM;
config.pin_xclk = XCLK_GPIO_NUM;
config.pin_pclk = PCLK_GPIO_NUM;
config.pin_vsync = VSYNC_GPIO_NUM;
config.pin_href = HREF_GPIO_NUM;
config.pin_sscb_sda = SIOD_GPIO_NUM;
config.pin_sscb_scl = SIOC_GPIO_NUM;
config.pin_pwdn = PWDN_GPIO_NUM;
config.pin_reset = RESET_GPIO_NUM;
config.xclk_freq_hz = 20000000;
config.pixel_format = PIXFORMAT_JPEG;
if(psramFound()){
config.frame_size = FRAMESIZE_UXGA;
config.jpeg_quality = 10;
config.fb_count = 2;
} else {
config.frame_size = FRAMESIZE_SVGA;
config.jpeg_quality = 12;
config.fb_count = 1;
}
// Camera init
esp_err_t err = esp_camera_init(&config);
if (err != ESP_OK) {
Serial.printf("Camera init failed with error 0x%x", err);
return;
}
// Wi-Fi connection
WiFi.begin(ssid, password);
while (WiFi.status() != WL_CONNECTED) {
delay(500);
Serial.print(".");
}
Serial.println("");
Serial.println("WiFi connected");
Serial.print("Camera Stream Ready! Go to: http://");
Serial.print(WiFi.localIP());
// Start streaming web server
startCameraServer();
}
void loop() {
delay(1);
}
Now my question is: How to control GPIO pins on ESP32 Camera, from the python script, so that when an object is detected, ESP32-Cam can activate an alarm system?
