Share My Creation Using the webcam to control the mouse

Virtual Mouse Using OpenCV (HERE)​

This project is using the live feed coming from the webcam to create a virtual mouse using hand tracking.
It is using the hand as a virtual mouse than can do everything that a mouse does without even touching your system. The webcam of the system is used to detect the hands. It will then create a bounding box around the hand and focus on two fingers: The fore finger and the middle finger. The fore finger will act as a cursor and moving it around, you will be moving the cursor around. Now, in order to successfully click using hand tracking, it is detecting the distance between the fore finger and the middle finger. If they are joined together, then it will perform a click. A smoothness factor was added to correct shaky movements.

 

[jkhazraji]

Sub Class_Globals
    Private Root As B4XView
    Private xui As XUI
    Public Py As PyBridge
End Sub

Public Sub Initialize
  
End Sub

'This event will be called once, before the page becomes visible.
Private Sub B4XPage_Created (Root1 As B4XView)
    Root = Root1
    Root.LoadLayout("MainPage")
    Py.Initialize(Me, "Py")
    Dim opt As PyOptions = Py.CreateOptions("Python/python/python.exe")
    Py.Start(opt)
    Wait For Py_Connected (Success As Boolean)
    If Success = False Then
        LogError("Failed to start Python process.")
        Return
    End If
    PrintPythonVersion
        VirtualMouse
End Sub

Private Sub B4XPage_Background
    Py.KillProcess
End Sub

Private Sub Py_Disconnected
    Log("PyBridge disconnected")
End Sub

Private Sub PrintPythonVersion
    Dim version As PyWrapper = Py.ImportModuleFrom("sys", "version")
    version.Print2("Python version:", "", False)
End Sub

Sub VirtualMouse
    'Virtual mouse using OpenCV
    'requirements
    'pip install mediapipe
    'pip install opencv-python
    'pip install autopy
    'copy "HandTracking.py" to your local Python shell
    Dim code As String=$"
import cv2
import numpy as np
import time
import HandTracking as ht
import autopy   # Install using "pip install autopy"

### Variables Declaration
pTime = 0               # Used to calculate frame rate
width = 640             # Width of Camera
height = 480            # Height of Camera
frameR = 100            # Frame Rate
smoothening = 8         # Smoothening Factor
prev_x, prev_y = 0, 0   # Previous coordinates
curr_x, curr_y = 0, 0   # Current coordinates

cap = cv2.VideoCapture(0)   # Getting video feed from the webcam
cap.set(3, width)           # Adjusting size
cap.set(4, height)

detector = ht.handDetector(maxHands=1)                  # Detecting one hand at max
screen_width, screen_height = autopy.screen.size()      # Getting the screen size
while True:
    success, img = cap.read()
    img = detector.findHands(img)                       # Finding the hand
    lmlist, bbox = detector.findPosition(img)           # Getting position of hand

    if len(lmlist)!=0:
        x1, y1 = lmlist[8][1:]
        x2, y2 = lmlist[12][1:]

        fingers = detector.fingersUp()      # Checking if fingers are upwards
        cv2.rectangle(img, (frameR, frameR), (width - frameR, height - frameR), (255, 0, 255), 2)   # Creating boundary box
        if fingers[1] == 1 and fingers[2] == 0:     # If fore finger is up and middle finger is down
            x3 = np.interp(x1, (frameR,width-frameR), (0,screen_width))
            y3 = np.interp(y1, (frameR, height-frameR), (0, screen_height))

            curr_x = prev_x + (x3 - prev_x)/smoothening
            curr_y = prev_y + (y3 - prev_y) / smoothening

            autopy.mouse.move(screen_width - curr_x, curr_y)    # Moving the cursor
            cv2.circle(img, (x1, y1), 7, (255, 0, 255), cv2.FILLED)
            prev_x, prev_y = curr_x, curr_y

        if fingers[1] == 1 and fingers[2] == 1:     # If fore finger & middle finger both are up
            length, img, lineInfo = detector.findDistance(8, 12, img)

            if length < 40:     # If both fingers are really close to each other
                cv2.circle(img, (lineInfo[4], lineInfo[5]), 15, (0, 255, 0), cv2.FILLED)
                autopy.mouse.click()    # Perform Click

    cTime = time.time()
    fps = 1/(cTime-pTime)
    pTime = cTime
    cv2.putText(img, str(int(fps)), (20, 50), cv2.FONT_HERSHEY_PLAIN, 3, (255, 0, 0), 3)
    cv2.imshow("Image", img)
    cv2.waitKey(1)
    "$
    Py.RunNoArgsCode(code)
End Sub