Nicla Vision receiving data from web page while streaming video

I am attempting to create a robot controlled by buttons on a web page served by a nicla vision.
I have set the nicla as a server in ap mode which creates a web page with streamed video and buttons to control the wheel motors.
As an initial test I have used the streaming sketch in openmv and added some buttons to the html . The buttons have been programmed to turn the blue LED on and off
The web page displays correctly as a streamed video window with 2 buttons displayed beneath.
All works until I press a button. The server disconnects briefly and only on reconnecting does the button press get processed. This can take several seconds. Is there a better way to receive data from the client. At present I have put the
client.recv(1024)
function inside the start_streaming(client) function.

Program


# This work is licensed under the MIT license.
# Copyright (c) 2013-2023 OpenMV LLC. All rights reserved.
# https://github.com/openmv/openmv/blob/master/LICENSE
#
# MJPEG Streaming AP.
#
# This example shows off how to do MJPEG streaming in AccessPoint mode.
# Chrome, Firefox and MJpegViewer App on Android have been tested.
# Connect to OPENMV_AP and use this URL: http://192.168.1.1:8080 to view the stream.

import sensor
import time
import network
import socket
from machine import LED, Pin

SSID = "OPENMV_AP"  # Network SSID
KEY = "1234567890"  # Network key (must be 10 chars)
HOST = ""  # Use first available interface
PORT = 8080  # Arbitrary non-privileged port

# Reset sensor
sensor.reset()
sensor.set_framesize(sensor.QQVGA)
sensor.set_pixformat(sensor.GRAYSCALE)

# Init wlan module in AP mode.
wlan = network.WLAN(network.AP_IF)
wlan.config(ssid=SSID, key=KEY, channel=2)
wlan.active(True)

print("AP mode started. SSID: {} IP: {}".format(SSID, wlan.ifconfig()[0]))

# You can block waiting for client to connect
# print(wlan.wait_for_sta(100000))

led = LED("LED_BLUE")  # Initialize LED (LED(1) is usually the red LED on the OpenMV Cam)


def start_streaming(client):
    
    # Send HTML page with control buttons
    html = (
        "HTTP/1.1 200 OK\r\n"
        "Content-Type: text/html\r\n"
        "Connection: close\r\n"
        "\r\n"
        "<html><head><title>OpenMV Cam</title></head><body>"
        "<form>"
        #"<button type=\"submit\" name=\"led\" value=\"on\">LED On</button>"
        #"<button type=\"submit\" name=\"led\" value=\"off\">LED Off</button>"
        "<button name=\"On\" type=\"submit\" value=\"On\">On</button>"
        "<button name=\"Off\" type=\"submit\" value=\"Off\">Off</button>"
        "</form>"
        "<h1>MJPEG Stream</h1>"
        "<img src=\"/stream\" style=\"width: 100%; max-width: 320px;\" />"
        "<h2>Control LED</h2>"

        "</body></html>"
    )

    if "/stream" in data:
        # Send multipart header
        client.send(
            "HTTP/1.1 200 OK\r\n"
            "Server: OpenMV\r\n"
            "Content-Type: multipart/x-mixed-replace;boundary=openmv\r\n"
            "Cache-Control: no-cache\r\n"
            "Pragma: no-cache\r\n\r\n"
        )

        # FPS clock
        clock = time.clock()
        start = time.ticks_ms()

        # Start streaming images
        # NOTE: Disable IDE preview to increase streaming FPS.
        while True:
            clock.tick()  # Track elapsed milliseconds between snapshots().
            frame = sensor.snapshot()
            cframe = frame.to_jpeg(quality=35, copy=True)
            header = (
                "\r\n--openmv\r\n"
                "Content-Type: image/jpeg\r\n"
                "Content-Length:" + str(cframe.size()) + "\r\n\r\n"
            )
            client.sendall(header)
            client.sendall(cframe)

            # Check if data received from client every sec
            delta = time.ticks_diff(time.ticks_ms(),start)
            if delta > 1000:
                start = time.ticks_ms()
               # get data from client
                data = client.recv(1024).decode('utf-8')
                print(data)
                 # Parse client request to control LED
                if "On" in data:
                    led.value(1)
                elif "Off" in data:
                    led.value(0)*

    else:
        # Send HTML page
        client.send(html)

    client.close()


server = None

while True:
    if server is None:
        # Create server socket
        server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
        # Bind and listen
        server.bind([HOST, PORT])
        server.listen(5)
        # Set server socket to blocking
        server.setblocking(True)

    try:
        print("Waiting for connections..")
        client, addr = server.accept()
    except OSError as e:
        server.close()
        server = None
        print("server socket error:", e)
        continue

    try:
        # set client socket timeout to 2s
        client.settimeout(5.0)
        print("Connected to " + addr[0] + ":" + str(addr[1]))
        start_streaming(client)
    except OSError as e:
        client.close()
        print("client socket error:", e)
        # sys.print_exception(e)

Hi, can you look into the async io class? We will be updating our examples to use it in the future. It has much better functionality for handling multiple threads of activity at a time.

asyncio — asynchronous I/O scheduler — MicroPython 1.23 documentation (openmv.io)

micropython-async/v3/docs/TUTORIAL.md at master · peterhinch/micropython-async (github.com)

Big tutorial above.

Here’s a whole guide: Raspberry Pi Pico W: Asynchronous Web Server (MicroPython) | Random Nerd Tutorials

Thanks for the suggestions. Async io looks the way to go.

Hi Kwabena,
I have tried using the asyncio class. Unfortunately I am still unable to stream video from the camera and responding to buttons at the same time. Its probably my lack of HTML knowledge that is letting me down. I’m not sure really how to go about updating the server with the image while reading the client data without having to close the client. I followed the code given by Rui Santos at Random Nerds and this worked fine with just buttons being read and basic data being sent to the client. I’m not sure how to send the header and image data to update the video. Can you give me any pointers.
Cheers Gary

Hi Nat,

I don’t have experience with this. However, I think you can use the image attribute to point to another URL on your device:

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Video Streaming</title>
</head>
<body>
    <h1>Video Streaming</h1>
    <img src="{{ url_for('video_feed') }}" width="640" height="480">
</body>
</html>

Then, you just need to handle the requests to get the mjpeg file and return a stream.

def _send_http_response(self, code, name, extra=""):  # private
        self._tcp_socket.send("HTTP/1.0 %d %s\r\n%s\r\n" % (code, name, extra))

def _send_http_response_ok(self, extra=""):  # private
    self._send_http_response(200, "OK", extra)

def _parse_mjpeg_request(self, data):  # private
    s = data.decode().splitlines()
    if s and len(s) >= 1:
        line0 = s[0].split(" ")
        request = line0[0]
        self._pathname = re.sub("(http://[a-zA-Z0-9\-\.]+(:\d+)?(/)?)", "/", line0[1])
        if self._pathname != "/" and self._pathname.endswith("/"):
            self._pathname = self._pathname[:-1]
        if line0[2] == "HTTP/1.0" or line0[2] == "HTTP/1.1":
            if request == "GET":
                self._playing = True
                self._send_http_response_ok(
                    "Server: OpenMV Cam\r\n"
                    "Content-Type: multipart/x-mixed-replace;boundary=" + self._boundary + "\r\n"
                    "Connection: close\r\n"
                    "Cache-Control: no-cache, no-store, must-revalidate\r\n"
                    "Pragma: no-cache\r\n"
                    "Expires: 0\r\n"
                    + ("\r\n".join(self._extra_headers) if len(self._extra_headers) else "")
                )
                if self._setup_cb:
                    self._setup_cb(self._pathname)
                return
            else:
                self._send_http_response(501, "Not Implemented")
                return
    self._send_http_response(400, "Bad Request")

def _send_mjpeg(self, image_callback, quality):  # private
    try:
        self._tcp_socket.settimeout(5)
        img = image_callback(self._pathname).to_jpeg(quality=quality)
        mjpeg_header = (
            "\r\n--" + self._boundary + "\r\n"
            "Content-Type: image/jpeg\r\n"
            "Content-Length:" + str(img.size()) + "\r\n\r\n"
        )
        self._tcp_socket.sendall(mjpeg_header)
        self._tcp_socket.sendall(img)
    except OSError:
        self._close_tcp_socket()

I pulled the code from this PR I need to switch to uasyncio: lib: Add mjpeg server. by kwagyeman · Pull Request #1982 · openmv/openmv (github.com). So, take the above code as inspiration.

Anyway, it should give you an idea of what you need to do. Which is that you have to handle a request for a path on the device for a file, and then when that request comes in you have to parse the request to see if it’s for an MJPEG file, and then return the data for it as an endless stream of jpegs.