B4J Question FFMpeg MJPEG Piping and showing to B4J frames at an imageview...

[Magma]

Sometimes, theoretically all works... but there are times (most)... that all going at different direction... so i am lowering the bar of the local/streaming/media...
to show for example a video as jpeg frames...

Here is my code (until now) - but there are some problems... according this:

JPEG File Interchange Format - Wikipedia

en.wikipedia.org

the SOI is a 2 byte 0xFF,0xD8 and the EOI is a 2byte of 0xFF,0xD9...
but when i am trying to extract those frame a jpeg - i am taking only 1Kb... to 2kb files... and a bad jpeg.. showing...

Sub Process_Globals
    Private fx As JFX
    Private MainForm As Form
    Private xui As XUI
    Private Button1 As B4XView
    Dim js As Shell
    Private Button2 As Button
    Dim check1for As B4XBytesBuilder
    Dim check2for As B4XBytesBuilder
    Private ImageView1 As ImageView
 
    Dim SOI(2) As Byte
    Dim EOI(2) As Byte
    Dim chunk As B4XBytesBuilder

    Dim s1 As Int=0
    Dim s11 As Int=0
End Sub

Sub AppStart (Form1 As Form, Args() As String)
    MainForm = Form1
    MainForm.RootPane.LoadLayout("Layout1")
    MainForm.Show
 
    SOI(0)=0xFF
    SOI(1)=0xD8
 
    EOI(0)=0xFF
    EOI(1)=0xD9
 
    chunk.Initialize

End Sub

Sub Button1_Click

    Dim par As List
    par.Initialize
 

    par.Add("-i")
    par.Add("1.avi")
    par.Add("-c:v")
    par.Add("mjpeg")
    par.Add("-f")
    par.Add("mjpeg")
    par.Add("-preset")
    par.Add("ultrafast")
    par.Add("-q")
    par.Add("20")
    par.Add("-r")
    par.Add("15")
    par.Add("-maxrate")
    par.Add("2M")
    par.add("-bufsize")
    par.Add("2M")
    par.Add("-")  'This creating a pipe..
    arun("ffmpeg.exe",par)
End Sub

Sub Button2_click
    js.KillProcess
End Sub

Sub arun(app As String,params As List)

    js.Initialize("js", app, params)
    js.WorkingDirectory=File.dirapp
    js.RunWithOutputEvents(-1)
End Sub

Sub js_StdOut (Buffer() As Byte, Length As Int)
 
    chunk.Append2(Buffer,0,Length)
 
    'If s1>-1 Then
    s1=chunk.IndexOf2(SOI,s11)
    If s1>-1 Then s11=s1'+1024
    'End If
    Dim s2 As Int = 0
    If s1>-1 Then
        Dim s2 As Int=chunk.IndexOf2(EOI,s1)
        If s2>s1 Then
            Dim aa As InputStream
            aa.InitializeFromBytesArray(chunk.ToArray,s1,(s2-s1)-2)
            Dim bmp As Image
            bmp.Initialize2(aa)
            ImageView1.SetImage(bmp)
            chunk.Clear 'this ofcourse may be remove a following frame... but not the only prob....
            Sleep(0)
            s1=0
            s11=0
        End If
    End If

end sub

Ofcourse this not work as i wanted... sometimes i am getting the first 10 pixels of a top frame....
but most i got: this: WARNING: Corrupt JPEG data: premature end of data segment

ps: some will tell why not using javacv-bin (with some java inline - is someway easy)... but with ffmpeg.exe i feel that i have more control... and i am clear to end users for future use in my apps that this way of capturing/streaming/playing is a 3rd tool... not mine

 

[Magma]

Hi there ... found a python code...
a conversion will be very helpful...

import cv2
import subprocess as sp
import numpy as np
import struct


IMG_W = 224
IMG_H = 224
input = 'test.mp4'

# Build synthetic video file for testing.
################################################################################
sp.run(['ffmpeg', '-y', '-f', 'lavfi', '-i', f'testsrc=size={IMG_W}x{IMG_H}:rate=1',
         '-f', 'lavfi', '-i', 'sine=frequency=300', '-c:v', 'libx264', '-pix_fmt', 'nv12',
         '-c:a', 'aac', '-ar', '22050', '-t', '50', input])
################################################################################

def read_from_pipe(p_stdout, n_bytes):
    """ Read n_bytes bytes from p_stdout pipe, and return the read data bytes. """
    data = p_stdout.read(n_bytes)
    while len(data) < n_bytes:
        data += p_stdout.read(n_bytes - len(data))

    return data


ffmpeg_cmd = ['ffmpeg', '-hwaccel', 'cuda', '-c:v', 'h264_cuvid', '-i', input, '-c:v', 'mjpeg', '-pix_fmt', 'yuvj420p', '-f', 'image2pipe', '-']

pipe = sp.Popen(ffmpeg_cmd, stdout=sp.PIPE)

jpg_list = []

cnt = 0
while True:
    if not pipe.poll() is None:
        break

    # https://en.wikipedia.org/wiki/JPEG_File_Interchange_Format
    jpeg_parts = []

    # SOI
    soi = read_from_pipe(pipe.stdout, 2)  # Read Start of Image (FF D8)
    assert soi == b'\xff\xd8', 'Error: first two bytes are not FF D8'
    jpeg_parts.append(soi)

    # JFIF APP0 marker segment
    marker = read_from_pipe(pipe.stdout, 2)  # APP0 marker (FF E0)
    assert marker == b'\xff\xe0', 'Error: APP0 marker is not FF E0'
    jpeg_parts.append(marker)

    xx = 0

    # Keep reading markers and segments until marker is EOI (0xFFD9)
    while xx != 0xD9:  # marker != b'\xff\xd9':
        # Length of segment excluding APP0 marker
        length_of_segment = read_from_pipe(pipe.stdout, 2)
        jpeg_parts.append(length_of_segment)
        length_of_segment = struct.unpack('>H', length_of_segment)[0]  # Unpack to uint16 (big endian)

        segment = read_from_pipe(pipe.stdout, length_of_segment - 2)  # Read the segment (minus 2 bytes because length includes the 2 bytes of length)
        jpeg_parts.append(segment)

        marker = read_from_pipe(pipe.stdout, 2)  # JFXX-APP0 marker (FF E0) or SOF or DHT or COM or SOS or EOI
        jpeg_parts.append(marker)

        if marker == b'\xff\xda':  # SOS marker (0xFFDA)
            # https://stackoverflow.com/questions/26715684/parsing-jpeg-sos-marker
            # Summary of how to find next marker after SOS marker (0xFFDA):
            #
            # Skip first 3 bytes after SOS marker (2 bytes header size + 1 byte number of image components in scan).
            # Search for next FFxx marker (skip every FF00 and range from FFD0 to FFD7 because they are part of scan).
            # *This is summary of comments below post of user3344003 + my knowledge + Table B.1 from https://www.w3.org/Graphics/JPEG/itu-t81.pdf.
            #
            # *Basing on Table B.1 I can also suspect that values FF01 and FF02 through FFBF should also be skipped in point 2 but I am not sure if they cannot appear as part of encoded SOS data.
            first3bytes = read_from_pipe(pipe.stdout, 3)
            jpeg_parts.append(first3bytes)  # Skip first 3 bytes after SOS marker (2 bytes header size + 1 byte number of image components in scan).

            xx = 0

            # Search for next FFxx marker, skip every FF00 and range from FFD0 to FFD7 and FF01 and FF02 through FFBF
            while (xx < 0xBF) or ((xx >= 0xD0) and (xx <= 0xD7)):
                # Search for next FFxx marker
                b = 0
                while b != 0xFF:
                    b = read_from_pipe(pipe.stdout, 1)
                    jpeg_parts.append(b)
                    b = b[0]
            
                xx = read_from_pipe(pipe.stdout, 1)  # Read next byte after FF
                jpeg_parts.append(xx)
                xx = xx[0]

    # Join list parts elements to bytes array, and append the bytes array to jpg_list (convert to NumPy array).
    jpg_list.append(np.frombuffer(b''.join(jpeg_parts), np.uint8))

    cnt += 1


pipe.stdout.close()
pipe.wait()


# Decode and show images for testing
for im in jpg_list:
    image = cv2.imdecode(im, cv2.IMREAD_UNCHANGED)

    cv2.imshow('test', image)
    if cv2.waitKey(100) & 0xFF == ord('q'):
        break

cv2.destroyAllWindows()

the link found.. caution seems like having some dangerous scripts as a webpage... hmmm