B4A Library Extracting Facial States and Facial Landmarks from a Bitmap (photo) using Google Mobile Vision API

Discussion in 'Additional libraries, classes and official updates' started by Johan Schoeman, May 2, 2016.

  1. Johan Schoeman

    Johan Schoeman Expert Licensed User

    It wraps this Github project. It extracts facial states and facial landmark information from a photo (bitmap) containing a human face. See the pic below...

    Posting the following:
    1. B4A project - b4aAndroidVisionPhotoFaceDetector.zip
    2. The B4A library files (AndroidVisionFaceViewLibFiles.zip)
    3. The java code as it stands at present (TheJavaCode.zip)
    4. A link to all the other jars that will be required - download them from here and copy them to your additional library folder - https://www.dropbox.com/s/se5x5znl1tgpv6r/OtherLibFiles.zip?dl=0
    5. resourceAdditionalLibFolder.zip - extract it and copy the resource folder to your additional library folder
    6. resource.zip - extract it and copy the resource folder to the same folder level that of the /Files and /Objects folders of the B4A project.

    You can download and test it but if you want to use it then you need to [​IMG]

    1.png

    Sample Code:
    Code:
    #Region  Project Attributes
        
    #ApplicationLabel: AndroidVisionPhotoFaceDetector
        
    #VersionCode: 1
        
    #VersionName:
        
    'SupportedOrientations possible values: unspecified, landscape or portrait.
        #SupportedOrientations: unspecified
        
    #CanInstallToExternalStorage: False
    #End Region

    #AdditionalRes: ..\resource
    #AdditionalRes: C:\Users\----------2\Documents\Basic 4 Android\JOHAN APPS\JHS LIBS\resource\b4a_appcompat, de.amberhome.objects.appcompat
    #AdditionalRes: C:\ANDRIOD_SDK_TOOLS\extras\android\support\v7\appcompat\res, android.support.v7.appcompat
    #AdditionalRes: C:\ANDRIOD_SDK_TOOLS\extras\google\google_play_services\libproject\google-play-services_lib\res, com.google.android.gms
    #AdditionalRes: C:\ANDRIOD_SDK_TOOLS\extras\android\support\design\res, android.support.design

    #ExcludeClasses: .games, .drive, .ads, .fitness, .wearable, .measurement, .cast, .auth, .nearby
    #ExcludeClasses: .tagmanager, .analytics, .wallet, .plus, .gcm, .maps, .panorama

    #Extends: android.support.v7.app.AppCompatActivity

    #Region  Activity Attributes
        
    #FullScreen: False
        
    #IncludeTitle: True
    #End Region

    Sub Process_Globals
        
    'These global variables will be declared once when the application starts.
        'These variables can be accessed from all modules.

    End Sub

    Sub Globals
        
    'These global variables will be redeclared each time the activity is created.
        'These variables can only be accessed from this module.

        
    Dim bm As Bitmap

        
    Private fv1 As AndroidVisionFaceView
        
    Private lv1 As ListView

    End Sub

    Sub Activity_Create(FirstTime As Boolean)
        
    'Do not forget to load the layout file created with the visual designer. For example:
        Activity.LoadLayout("main")

        lv1.SingleLineLayout.Label.TextSize = 
    15
        lv1.SingleLineLayout.Label.TextColor = 
    Colors.White
        lv1.TwoLinesLayout.Label.TextSize = 
    15
        lv1.TwoLinesLayout.Label.TextColor = 
    Colors.white
        bm.Initialize(
    File.DirAssets,"charlize.jpg")
        fv1.Photo = bm

        fv1.CircleColor = 
    Colors.Magenta
        fv1.CircleWidth = 
    3
        fv1.CircleDiameter = 
    5

        fv1.build

    End Sub

    Sub Activity_Resume

    End Sub

    Sub Activity_Pause (UserClosed As Boolean)

    End Sub

    Sub fv1_face_found(LandMarks As Object, IsLeftEyeOpenProbability As Float, IsRightEyeOpenProbability As Float, _
                       IsSmilingProbability 
    As Float, EulerY As Float, EulerZ As Float, iD As Int)


        lv1.AddSingleLine(
    "face ID = " & iD)
        lv1.AddTwoLines(
    "IsLeftEyeOpenProbability:""" & IsLeftEyeOpenProbability)
        lv1.AddTwoLines(
    "IsRightEyeOpenProbability:""" & IsRightEyeOpenProbability)
        lv1.AddTwoLines(
    "IsSmilingProbability:""" & IsSmilingProbability)
        lv1.AddTwoLines(
    "Rotation of face around vertical(Y) axis:""" & EulerY)
        lv1.AddTwoLines(
    "Rotation of face around Z axis:""" & EulerZ)

        
    Dim mylist As List
        mylist = LandMarks
        
    For i = 0 To mylist.Size - 1
             
    If mylist.Get(i) = "0" Then lv1.AddSingleLine("Landmark found at BOTTOM_MOUTH")
          
    If mylist.Get(i) = "1" Then lv1.AddSingleLine("Landmark found at LEFT_CHEEK")
          
    If mylist.Get(i) = "2" Then lv1.AddSingleLine("Landmark found at LEFT_EAR_TIP")
          
    If mylist.Get(i) = "3" Then lv1.AddSingleLine("Landmark found at LEFT_EAR")
          
    If mylist.Get(i) = "4" Then lv1.AddSingleLine("Landmark found at LEFT_EYE")
          
    If mylist.Get(i) = "5" Then lv1.AddSingleLine("Landmark found at LEFT_MOUTH")
          
    If mylist.Get(i) = "6" Then lv1.AddSingleLine("Landmark found at NOSE_BASE")
          
    If mylist.Get(i) = "7" Then lv1.AddSingleLine("Landmark found at RIGHT_CHEEK")
          
    If mylist.Get(i) = "8" Then lv1.AddSingleLine("Landmark found at RIGHT_EAR_TIP")
          
    If mylist.Get(i) = "9" Then lv1.AddSingleLine("Landmark found at RIGHT_EAR")
          
    If mylist.Get(i) = "10" Then lv1.AddSingleLine("Landmark found at RIGHT_EYE")
          
    If mylist.Get(i) = "11" Then lv1.AddSingleLine("Landmark found at RIGHT_MOUTH")

        
    Next

        lv1.AddSingleLine(
    "*********************************")

    '  AN EXPLANATION FOR THE LANDMARK STRING VALUES THAT ARE RETURNED TO THE B4A LIST
    '  Public static final int BOTTOM_MOUTH = 0;
    '  Public static final int LEFT_CHEEK = 1;
    '  Public static final int LEFT_EAR_TIP = 2;
    '  Public static final int LEFT_EAR = 3;
    '  Public static final int LEFT_EYE = 4;
    '  Public static final int LEFT_MOUTH = 5;
    '  Public static final int NOSE_BASE = 6;
    '  Public static final int RIGHT_CHEEK = 7;
    '  Public static final int RIGHT_EAR_TIP = 8;
    '  Public static final int RIGHT_EAR = 9;
    '  Public static final int RIGHT_EYE = 10;
    '  Public static final int RIGHT_MOUTH = 11;


    End Sub
    Library:
    AndroidVisionFaceView
    Author:
    Github: Clayton Wilkinson, Wrapped by: Johan Schoeman
    Version: 1
    AndroidVisionFaceView
    Events:

    • face_found (LandMarks As Object, IsLeftEyeOpenProbability As Float, IsRightEyeOpenProbability As Float, IsSmilingProbability As Float, EulerY As Float, EulerZ As Float, iD As Int)
    Fields:
    • ALL_LANDMARKS As Int
    • NO_LANDMARKS As Int
    • ba As BA
    Methods:
    • BringToFront
    • DesignerCreateView (base As PanelWrapper, lw As LabelWrapper, props As Map)
    • Initialize (EventName As String)
    • Invalidate
    • Invalidate2 (arg0 As Rect)
    • Invalidate3 (arg0 As Int, arg1 As Int, arg2 As Int, arg3 As Int)
    • IsInitialized As Boolean
    • RemoveView
    • RequestFocus As Boolean
    • SendToBack
    • SetBackgroundImage (arg0 As Bitmap)
    • SetColorAnimated (arg0 As Int, arg1 As Int, arg2 As Int)
    • SetLayout (arg0 As Int, arg1 As Int, arg2 As Int, arg3 As Int)
    • SetLayoutAnimated (arg0 As Int, arg1 As Int, arg2 As Int, arg3 As Int, arg4 As Int)
    • SetVisibleAnimated (arg0 As Int, arg1 As Boolean)
    • build
    Permissions:
    • android.hardware.camera
    • android.hardware.camera.autofocus
    • android.permission.CAMERA
    • android.permission.FLASHLIGHT
    • android.permission.VIBRATE
    • android.permission.WRITE_EXTERNAL_STORAGE
    Properties:
    • Background As Drawable
    • CircleColor As Int [write only]
    • CircleDiameter As Int [write only]
    • CircleWidth As Int [write only]
    • Color As Int [write only]
    • Enabled As Boolean
    • Height As Int
    • LandmarkType As Int [write only]
    • Left As Int
    • MinFaceSize As Float [write only]
    • Parent As Object [read only]
    • Photo As Bitmap [write only]
    • ProminentFaceOnly As Boolean [write only]
    • Tag As Object
    • Top As Int
    • Visible As Boolean
    • Width As Int
    Important: Take note of all the files in the B4A project's /Objects/res/blabla folders. Make sure they are set to READ ONLY before you compile the project for the first time

    Also important to note the B4A project's Manifest Files

    And equally important to set your paths correctly (the below only applicable to my folder setup):

    Code:
    #AdditionalRes: ..\resource[/I][/B]
    [B][I]
    #AdditionalRes: C:\Users\----------2\Documents\Basic 4 Android\JOHAN APPS\JHS LIBS\resource\b4a_appcompat, de.amberhome.objects.appcompat
    #AdditionalRes: C:\ANDRIOD_SDK_TOOLS\extras\android\support\v7\appcompat\res, android.support.v7.appcompat
    #AdditionalRes: C:\ANDRIOD_SDK_TOOLS\extras\google\google_play_services\libproject\google-play-services_lib\res, com.google.android.gms
    #AdditionalRes: C:\ANDRIOD_SDK_TOOLS\extras\android\support\design\res, android.support.design

     

    Attached Files:

    Last edited: May 4, 2016
    scsjc, Almora, tigleth and 13 others like this.
  2. DonManfred

    DonManfred Expert Licensed User

    Cool. Thank you for sharing!
     
    Johan Schoeman likes this.
  3. MarcoRome

    MarcoRome Expert Licensed User

    Another GREAT GREAT Library +50 ;)
     
    Johan Schoeman likes this.
  4. LucaMs

    LucaMs Expert Licensed User

  5. LucaMs

    LucaMs Expert Licensed User

    Is there a check:

    "Is she's cute/nice/beautiful? if yes, send her my phone number"

    ?

    :)
     
    Last edited: Sep 6, 2016
    Rantor777 likes this.
  6. MarcoRome

    MarcoRome Expert Licensed User

    To better understand the work Johan looks also HERE
     
    LucaMs likes this.
  7. Johan Schoeman

    Johan Schoeman Expert Licensed User

    Posting an update:

    1. New B4A project that will accommodate multiple faces in the same photo
    2. New B4A library files

    Follow the setup in post #1 and then replace the B4A project and B4A library files with the attached. Take note of point #6 in post #1:
    ".....6. resource.zip - extract it and copy the resource folder to the same folder level that of the /Files and /Objects folders of the B4A project......"

    The list at the bottom will show the ID(s) of the face(s) and it's various extracted facial landmarks and states. It will by default draw the ID of the face on the pic (you can choose to draw the ID or not draw the ID) so that you can relate the info in the list with each face in a multi-face picture.

    You can download and test it but if you want to use it then you need to [​IMG]

    2.png

    Sample Code (take note of the order of extraction from the list):
    Code:
    #Region  Project Attributes
        
    #ApplicationLabel: AndroidVisionPhotoFaceDetector
        
    #VersionCode: 1
        
    #VersionName:
        
    'SupportedOrientations possible values: unspecified, landscape or portrait.
        #SupportedOrientations: unspecified
        
    #CanInstallToExternalStorage: False
    #End Region

    #AdditionalRes: ..\resource

    #AdditionalRes: C:\Users\----------2\Documents\Basic 4 Android\JOHAN APPS\JHS LIBS\resource\b4a_appcompat, de.amberhome.objects.appcompat
    #AdditionalRes: C:\ANDRIOD_SDK_TOOLS\extras\android\support\v7\appcompat\res, android.support.v7.appcompat
    #AdditionalRes: C:\ANDRIOD_SDK_TOOLS\extras\google\google_play_services\libproject\google-play-services_lib\res, com.google.android.gms
    #AdditionalRes: C:\ANDRIOD_SDK_TOOLS\extras\android\support\design\res, android.support.design

    #ExcludeClasses: .games, .drive, .ads, .fitness, .wearable, .measurement, .cast, .auth, .nearby
    #ExcludeClasses: .tagmanager, .analytics, .wallet, .plus, .gcm, .maps, .panorama

    #Extends: android.support.v7.app.AppCompatActivity

    #Region  Activity Attributes
        
    #FullScreen: False
        
    #IncludeTitle: True
    #End Region

    Sub Process_Globals
        
    'These global variables will be declared once when the application starts.
        'These variables can be accessed from all modules.

    End Sub

    Sub Globals
        
    'These global variables will be redeclared each time the activity is created.
        'These variables can only be accessed from this module.

        
    Dim bm As Bitmap

        
    Private fv1 As AndroidVisionFaceView
        
    Private lv1 As ListView
        
    Dim sf As StringFunctions

    End Sub

    Sub Activity_Create(FirstTime As Boolean)
        
    'Do not forget to load the layout file created with the visual designer. For example:
        Activity.LoadLayout("main")

        sf.Initialize

        lv1.SingleLineLayout.Label.TextSize = 
    15
        lv1.SingleLineLayout.Label.TextColor = 
    Colors.White
        lv1.TwoLinesLayout.Label.TextSize = 
    15
        lv1.TwoLinesLayout.Label.TextColor = 
    Colors.white
        bm.Initialize(
    File.DirAssets,"three.jpg")
        fv1.Photo = bm

        fv1.CircleColor = 
    Colors.Cyan
        fv1.CircleWidth = 
    3
        fv1.CircleDiameter = 
    5

        fv1.MinFaceSize = 
    0.1                   
        fv1.ShowFaceIds = 
    True

        fv1.build

    End Sub

    Sub Activity_Resume

    End Sub

    Sub Activity_Pause (UserClosed As Boolean)

    End Sub

    Sub fv1_face_found(LandMarks As Object)
       
        
    Dim mylist As List
        mylist = LandMarks
        
    Dim lm As Int = sf.Val(mylist.Get(mylist.Size - 1))
        lv1.AddSingleLine(
    "face ID = " & mylist.Get(mylist.Size - 2))
        
    For i = 0 To lm - 1
          
    If mylist.Get(i) = "0" Then lv1.AddSingleLine("Landmark found at BOTTOM_MOUTH")
          
    If mylist.Get(i) = "1" Then lv1.AddSingleLine("Landmark found at LEFT_CHEEK")
          
    If mylist.Get(i) = "2" Then lv1.AddSingleLine("Landmark found at LEFT_EAR_TIP")
          
    If mylist.Get(i) = "3" Then lv1.AddSingleLine("Landmark found at LEFT_EAR")
          
    If mylist.Get(i) = "4" Then lv1.AddSingleLine("Landmark found at LEFT_EYE")
          
    If mylist.Get(i) = "5" Then lv1.AddSingleLine("Landmark found at LEFT_MOUTH")
          
    If mylist.Get(i) = "6" Then lv1.AddSingleLine("Landmark found at NOSE_BASE")
          
    If mylist.Get(i) = "7" Then lv1.AddSingleLine("Landmark found at RIGHT_CHEEK")
          
    If mylist.Get(i) = "8" Then lv1.AddSingleLine("Landmark found at RIGHT_EAR_TIP")
          
    If mylist.Get(i) = "9" Then lv1.AddSingleLine("Landmark found at RIGHT_EAR")
          
    If mylist.Get(i) = "10" Then lv1.AddSingleLine("Landmark found at RIGHT_EYE")
          
    If mylist.Get(i) = "11" Then lv1.AddSingleLine("Landmark found at RIGHT_MOUTH")
        
    Next
        lv1.AddTwoLines(
    "Rotation of face around vertical(Y) axis:""" & mylist.Get(i))
        lv1.AddTwoLines(
    "Rotation of face around Z axis:""" & mylist.Get(i+1))
        lv1.AddTwoLines(
    "IsLeftEyeOpenProbability:""" & mylist.Get(i+2))
        lv1.AddTwoLines(
    "IsRightEyeOpenProbability:""" & mylist.Get(i+3))
        lv1.AddTwoLines(
    "IsSmilingProbability:""" & mylist.Get(i+4))
        lv1.AddSingleLine(
    "No of LandMarks = " & mylist.Get(i+6))
        lv1.AddSingleLine(
    "*********************************")

    End Sub
    Library as it stands now:

    AndroidVisionFaceView


    Author: Github: Clayton Wilkinson, Wrapped by: Johan Schoeman
    Version: 1
    • AndroidVisionFaceView
      Events:
      • face_found (LandMarks As Object)
      Fields:
      • ALL_LANDMARKS As Int
      • NO_LANDMARKS As Int
      • ba As BA
      Methods:
      • BringToFront
      • DesignerCreateView (base As PanelWrapper, lw As LabelWrapper, props As Map)
      • Initialize (EventName As String)
      • Invalidate
      • Invalidate2 (arg0 As Rect)
      • Invalidate3 (arg0 As Int, arg1 As Int, arg2 As Int, arg3 As Int)
      • IsInitialized As Boolean
      • RemoveView
      • RequestFocus As Boolean
      • SendToBack
      • SetBackgroundImage (arg0 As Bitmap)
      • SetColorAnimated (arg0 As Int, arg1 As Int, arg2 As Int)
      • SetLayout (arg0 As Int, arg1 As Int, arg2 As Int, arg3 As Int)
      • SetLayoutAnimated (arg0 As Int, arg1 As Int, arg2 As Int, arg3 As Int, arg4 As Int)
      • SetVisibleAnimated (arg0 As Int, arg1 As Boolean)
      • build
        Build the facial state and landmarks based on initial settings
      Permissions:
      • android.hardware.camera
      • android.hardware.camera.autofocus
      • android.permission.CAMERA
      • android.permission.FLASHLIGHT
      • android.permission.VIBRATE
      • android.permission.WRITE_EXTERNAL_STORAGE
      Properties:
      • Background As Drawable
      • CircleColor As Int [write only]
        Set the landmark circle color
        The default color is Green
      • CircleDiameter As Int [write only]
        Set the landmark circle diameter
        The default diameter is 10
      • CircleWidth As Int [write only]
        Set the landmark circle width
        The default with is 5
      • Color As Int [write only]
      • Enabled As Boolean
      • Height As Int
      • LandmarkType As Int [write only]
        Set the landmark type
        0 = No Landmarks
        1 = All Landmarks
        Default value is 1 (= Show All Landmarks)
      • Left As Int
      • MinFaceSize As Float [write only]
        Set the minimum face size
        A float value from 0 and 1 - especially when using a photo with more than one face
      • Parent As Object [read only]
      • Photo As Bitmap [write only]
        Set the bitmap (photo containing human faces) to be analyzed
      • ProminentFaceOnly As Boolean [write only]
        Analyze the dominate face only?
        Default value is false
      • ShowFaceIds As Boolean [write only]
        Draw the face ID's of the analysis in the view?
        Default value is true
      • Tag As Object
      • Top As Int
      • Visible As Boolean
      • Width As Int
     

    Attached Files:

    Last edited: May 4, 2016
  8. Johan Schoeman

    Johan Schoeman Expert Licensed User

    Do you want them both (face and barcode) recognized in the same camera view?
     
    MarcoRome likes this.
  9. MarcoRome

    MarcoRome Expert Licensed User

    :D
     
  10. Johan Schoeman

    Johan Schoeman Expert Licensed User

    Do you want to capture prisoners? ;)
     
    LucaMs and MarcoRome like this.
  11. MarcoRome

    MarcoRome Expert Licensed User

    Never say never ;)
     
    Johan Schoeman likes this.
  12. Johan Schoeman

    Johan Schoeman Expert Licensed User

    Sometime in this week...
     
    MarcoRome likes this.
  13. Johan Schoeman

    Johan Schoeman Expert Licensed User

    Some refinement to do in this week but it is working (multi face and multi barcodes recognised simultaneously via the camera)....will post the lib this weekend ;)

    1.png
     
    Last edited: May 4, 2016
    MarcoRome, inakigarm and DonManfred like this.
  14. MarcoRome

    MarcoRome Expert Licensed User

    +60 ;) of course
     
    Johan Schoeman likes this.
  15. strat

    strat Active Member Licensed User

    Johan thank you for this lib.
    I get error when I compiled both b4aAndroidVisionPhotoFaceDetector and b4aAndroidVisionFaceTracker project. My android version is 4.4 and Appcompat example is working. I couldn't find solution on the net.



    LogCat connected to: b2045e08
    --------- beginning of /dev/log/system--------- beginning of /dev/log/main
    ** Service (starter) Create **
    ** Service (starter) Start **
    ** Activity (main) Create, isFirst = true **
    ID of Front Facing Camera = 1
    ID of Back Facing Camera = 0
    Device has a flash = true
    ** Activity (main) Resume **
    ** Activity (main) Pause, UserClosed = false **
    ** Activity (main_scanner) Create, isFirst = true **
    ** Activity (main_scanner) Resume **
    ** Activity (main_scanner) Pause, UserClosed = false **
    java.lang.NoClassDefFoundError: com.google.android.gms.vision.face.FaceDetector$Builder

    at main.java.com.google.android.gms.samples.vision.face.facetracker.FaceTrackerActivity.createCameraSource(FaceTrackerActivity.java:127)

    at main.java.com.google.android.gms.samples.vision.face.facetracker.FaceTrackerActivity.onCreate(FaceTrackerActivity.java:81)

    at android.app.Activity.performCreate(Activity.java:5231)

    at android.app.Instrumentation.callActivityOnCreate(Instrumentation.java:1087)

    at android.app.ActivityThread.performLaunchActivity(ActivityThread.java:2169)

    at android.app.ActivityThread.handleLaunchActivity(ActivityThread.java:2271)

    at android.app.ActivityThread.access$800(ActivityThread.java:144)

    at android.app.ActivityThread$H.handleMessage(ActivityThread.java:1205)

    at android.os.Handler.dispatchMessage(Handler.java:102)

    at android.os.Looper.loop(Looper.java:136)

    at android.app.ActivityThread.main(ActivityThread.java:5146)

    at java.lang.reflect.Method.invokeNative(Native Method)

    at java.lang.reflect.Method.invoke(Method.java:515)

    at com.android.internal.os.ZygoteInit$MethodAndArgsCaller.run(ZygoteInit.java:796)

    at com.android.internal.os.ZygoteInit.main(ZygoteInit.java:612)

    at dalvik.system.NativeStart.main(Native Method)
     
  16. Johan Schoeman

    Johan Schoeman Expert Licensed User

    Do you have the latest google-play-services.jar in your additional library folder?

     
    strat likes this.
  17. strat

    strat Active Member Licensed User

    Thank you. It was missing.
     
    Johan Schoeman likes this.
  18. danoptic

    danoptic Member Licensed User

    i cant get the
    #AdditionalRes
    working
    using B4a 6 and windows 10 and last sdk
     
  19. Johan Schoeman

    Johan Schoeman Expert Licensed User

    I have not tried to do with the latest SDK and with B4A 6. I am still trying to get to grips with the new SDK and everything that needs to change to make it work....
     
  20. danoptic

    danoptic Member Licensed User

    so witch SDK to use?

    this is the full #additionalres. with what to replace it?

    '#AdditionalRes: ..\resource\

    #AdditionalRes: C:\Users\Owner\Dropbox\basic4android\SourceCode\b4aAndroidVisionFaceTracker\resource\b4a_appcompat, de.amberhome.objects.appcompat
    '#AdditionalRes: C:\ANDRIOD_SDK_TOOLS\extras\android\support\v7\appcompat\res, android.support.v7.appcompat
    #AdditionalRes: C:\Users\Owner\AppData\Local\Android\android-sdk\extras\google\m2repository\com\google\android\gms, com.google.android.gms
    '#AdditionalRes: C:\ANDRIOD_SDK_TOOLS\extras\android\support\design\res, android.support.design
     
    Last edited: Aug 22, 2016
Loading...
  1. This site uses cookies to help personalise content, tailor your experience and to keep you logged in if you register.
    By continuing to use this site, you are consenting to our use of cookies.
    Dismiss Notice