B4J Question Uploading part of a file to OpenAI

Tim Chapman

Active Member
Licensed User
Longtime User
I have been fighting with this problem for 2 days.
I have checked every google search result I can find.

I am trying to follow the instructions here: https://platform.openai.com/docs/api-reference/uploads/add-part

The curl command to upload a part of a file is here:

curl https://api.openai.com/v1/uploads/upload_abc123/parts
-F data="aHR0cHM6Ly9hcGkub3BlbmFpLmNvbS92MS91cGxvYWRz..."

I have used Perplexity AI and OpenAI to try to convert that to B4J code correctly.
I have use this converter to convert it to okhttp which B4J supports:
https://curlconverter.com/java-okhttp/

I have put the response from there into Perplexity and OpenAI to try to get workable code.
With innumerable iterations and changes, I have not gotten it to work.

The code works properly to initialize the upload of parts, but the uploading of the parts themselves returns an error.

The closest I have come is this code:

B4J code to upload part of a file to OpenAI:
#Region Shared Files
#CustomBuildAction: folders ready, %WINDIR%\System32\Robocopy.exe,"..\..\Shared Files" "..\Files" ' This line defines a custom build action that uses Robocopy to synchronize files from the "Shared Files" directory to the "Files" directory.
'Ctrl + click to sync files: ide://run?file=%WINDIR%\System32\Robocopy.exe&args=..\..\Shared+Files&args=..\Files&FilesSync=True ' This line provides a clickable link in the IDE to manually trigger the file synchronization using Robocopy.
#End Region

'Ctrl + click to export as zip: ide://run?File=%B4X%\Zipper.jar&Args=Project.zip ' This line provides a clickable link in the IDE to export the project as a zip file using Zipper.jar.

Sub Class_Globals
    Private Root As B4XView ' Root view of the page.
    Private xui As XUI ' Cross-platform UI library.
    Private OpenAIapiKey As String = "My API Key is in the attached zip file.  Please don't abuse it."
    Private Label1 As Label
End Sub

Public Sub Initialize ' Initialization method for the class.
    Button1_Click
    Label1.Initialize("")
    Label1.Text = "Please watch the logs.  It takes 5 or 10 minutes to get the final response." & CRLF & "See line 30 and change it to a folder you like and line 36 to a file you like." & CRLF & "Also, see line 70 of the code.  I limited it to 1 part becuse I run out of memory otherwise.   I will fix that after I get an appropriate response from a part upload."
End Sub

'This event will be called once, before the page becomes visible.
Private Sub B4XPage_Created (Root1 As B4XView)
    Root = Root1 ' Assign the root view.
    Root.LoadLayout("MainPage") ' Load the layout named "MainPage".
End Sub

'You can see the list of page related events in the B4XPagesManager object. The event name is B4XPage.

Private Sub Button1_Click ' Event handler for Button1 click.
    Private uploadApiUrl As String = "https://api.openai.com/v1/uploads"
    Dim Folder As String = "D:\Downloads"
  
    Log("File.DirApp = " & File.DirApp)
    Log("File.DirAssets = " & File.DirAssets) ' This is the Files Folder.
    Log("File.DirTemp = " & File.DirTemp)
      
    Dim FileName As String = "uploadtest.zip"
    Dim Purpose As String = "assistants"
  
    Dim Job As HttpJob
    Job.Initialize("initializeUploadJob", Me)
    Dim FilePath As String = Folder & "\" & FileName
    Job.Tag = FilePath
    Dim fileSize As Long = File.Size(Folder, FileName)
    Dim mimeType As String = GetMimeType1(FileName) ' Adjust MIME type as needed
    Dim json As String = $"{"filename": "${FileName}", "purpose": "${Purpose}", "bytes": ${fileSize}, "mime_type": "${mimeType}"}"$

    Job.PostString(uploadApiUrl, json)
    Job.GetRequest.SetHeader("Authorization", "Bearer " & OpenAIapiKey)
    Job.GetRequest.SetContentType("application/json")
    Job.GetRequest.SetHeader("OpenAI-Beta", "assistants=v2")
  
    Wait For (Job) JobDone (Job As HttpJob) ' Wait for the job to complete.
  
    If Job.Success Then
        LogColor("Job = " & Job,0xFF006D11) ' Log the job
        Log("Initialize Upload Job Response = " & Job.GetString) ' Log the job result.
  
        ' Convert JSON string to map
        Dim parser As JSONParser
        parser.Initialize(Job.GetString)
        Dim result As Map = parser.NextObject
  
        Dim FileID As String = result.Get("id")
        Log("File ID = " & FileID)

        ' Proceed to upload parts
        Dim fileSize As Long = File.Size(Folder, FileName)
        Log("File Size = " & fileSize)
        Dim partSize As Long = 64 * 1024 * 1024 ' 64 MB
        Dim partCount As Int = Ceil(fileSize / partSize)
        Log("PartCount = " & partCount)
        For i = 0 To 1 'partCount - 1
            Dim start As Long = i * partSize
            Dim end1 As Long = Min((i + 1) * partSize, fileSize)
            'Log("Uploadpart called: " & start & "," & end1 & "," & Folder & "," &  FileName & "," &  FileID & "," &  apiKey)
            LogColor("***",0xFFFF3700)

            ' *** THE CODE WORKS PROPERLY UP TO HERE ***

            'Upload Part
            Dim partJob As HttpJob
            partJob.Initialize("uploadPartJob", Me)
  
            Dim url As String = "https://api.openai.com/v1/uploads/" & FileID & "/parts"
  
            Dim length As Long = end1 - start
            Dim raf As RandomAccessFile
            raf.Initialize(Folder, FileName, False)
            Log("Uploading " & length & " bytes")
  
            Dim partData(length) As Byte
            raf.ReadBytes(partData, 0, length, start)
            raf.Close
  
            ' Create a MultipartFileData for the part data
            Dim mfd As MultipartFileData
            mfd.Initialize
            mfd.Dir = Folder ' No directory needed since we're using raw bytes
            mfd.FileName = FileName ' No file name needed
            mfd.KeyName = "data" ' The key expected by the server
            mfd.ContentType = "multipart/form-data" ' Set appropriate content type
  
            ' Create a map for the multipart form data
            Dim multipartData As Map
            multipartData.Initialize
            multipartData.Put("data", partData)
  
            ' Post the multipart request
            partJob.PostMultipart(url, multipartData, Array(mfd))
            partJob.GetRequest.SetHeader("Authorization", "Bearer " & OpenAIapiKey)
            partJob.GetRequest.SetContentType("multipart/form-data")
            LogColor("Part Job = " & partJob,0xFF006D11) ' Log request for troubleshooting
          
            Wait For (Job) JobDone (Job As HttpJob) ' Wait for the job to complete.
          
            LogColor("PartUpload Job Response = " & Job.GetString,0xFFFF3700) ' Log the job result.
          
            If Job.Success Then
  
                ' Convert JSON string to map
                Dim parser As JSONParser
                parser.Initialize(Job.GetString)
                Dim result As Map = parser.NextObject
  
                Dim FileID As String = result.Get("id")
                Log("File ID = " & FileID)
            Else
                ' Call error sub for error handling
                xui.MsgboxAsync(Job.JobName & ": " & Job.ErrorMessage,"Error")
            End If
            Job.Release
        Next
    Else
        ' Call error sub for error handling
        xui.MsgboxAsync(Job.JobName & ": " & Job.ErrorMessage,"Error")
    End If
    Job.Release
End Sub

Private Sub GetMimeType1(fileName As String) As String
    Dim extension As String = fileName.SubString(fileName.LastIndexOf(".") + 1).ToLowerCase
    Select extension
        Case "txt"
            Return "text/plain"
        Case "html", "htm"
            Return "text/html"
        Case "jpg", "jpeg"
            Return "image/jpeg"
        Case "png"
            Return "image/png"
        Case "pdf"
            Return "application/pdf"
        Case "doc"
            Return "application/msword"
        Case "docx"
            Return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
        Case "xls"
            Return "application/vnd.ms-excel"
        Case "xlsx"
            Return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
        Case Else
            Return "application/octet-stream"
    End Select
End Sub

For all other attempts, I have gotten a response that the server can’t understand the request.
For the above code I got this error:

09/10/2024 12:19:12 - OpenAI Error: uploadPartJob: {
"error": {
"message": "'data' is a required property",
"type": "invalid_request_error",
"param": null,
"code": null
}
}

Even though I have the “data” property in the request, I still get that error as you can see.
There are no examples online that I can find for doing multi-part uploads. There is plenty of examples about uploading files of 512MB or less, but nothing about multi-part uploads (which are required for files greater than 512 MB).

I have tried to find examples here to convert to B4J as well with no luck: https://github.com/openai/openai-python/tree/main

I would sure appreciate any help you can give on this. Attached is a small program to show the problem. It has my OpenAI key in it so please don't abuse it. I will disable that key when this is resolved.
 

Attachments

  • PartUploadTest.zip
    4.8 KB · Views: 34
Last edited:
Solution
I got it working. Code is below.

Add your API key, folder and file to upload on lines 4, 23 and 24.

I will have this in the OpenAI library soon.

Thank you everyone for the help!

B4J upload file in parts to OpenAI:
Sub Class_Globals
    Private Root As B4XView ' Root view of the page.
    Private xui As XUI ' Cross-platform UI library.
    Private OpenAIapiKey As String = "YOUR_API_KEY"
    Private Label1 As Label
End Sub

Public Sub Initialize ' Initialization method for the class.
    Button1_Click
End Sub

'This event will be called once, before the page becomes visible.
Private Sub B4XPage_Created (Root1 As B4XView)
    Root = Root1 ' Assign the root view.
    Root.LoadLayout("MainPage") ' Load the layout named "MainPage".
    'Label1.Text = "Please watch the...

Erel

B4X founder
Staff member
Licensed User
Longtime User
This is not the correct way to build json strings.

B4X:
Dim m As Map = CreateMap("filename": FileName, "purpose": Purpose, ...)
Dim json As String = m.As(JSON).ToString

This cannot work:
B4X:
  Dim multipartData As Map
            multipartData.Initialize
            multipartData.Put("data", partData)

It will be converted to a meaningless string.

Save the data to a file first.
 
Upvote 0

LucaMs

Expert
Licensed User
Longtime User
Maybe I'm wrong but I think that in this case you can't use PostMultipart, you have to send a request for each block.
Am I wrong?


(BTW you should remove the initialization of Label1, set its text after LoadLayout and then call Button1_Click)
 
Upvote 0

Tim Chapman

Active Member
Licensed User
Longtime User
I apologize but I need more help on this. If I understand Erel correctly, I have to save the partial first? Then how do I proceed?
 
Upvote 0

Tim Chapman

Active Member
Licensed User
Longtime User
Because I already have that functionality working perfectly. I am making a comprehensive library for B4A/B4J that will do all the functions on that entire API reference page as well as tools. I have found this library (https://github.com/openai/openai-python) which is very helpful if you know python. So, I am uploading all 316 files in the src folder to Perplexity AI as a one text file with the file path for each file before the file's text in the large text file. Then I will have Perplexity AI convert the entire python library to B4J. It would be really helpful if I could do similarly with the entire B4J and B4A docs and libraries so that Perplexity (and later OpenAI) would make less errors in making code. If you will give me a copy of the b4x website, I will sort it out and make that happen. Or, if you have a subset of your entire website that you know to be relevant for my purposes, I would happily do so with that.
 
Upvote 0

Tim Chapman

Active Member
Licensed User
Longtime User
The upload parts functionality is to allow the uploading of files larger than 512 MB to OpenAI. Up to 10GB. I just came up with that approach to this problem after consulting with my twin brother who has been programming python for many years.
 
Upvote 0

Tim Chapman

Active Member
Licensed User
Longtime User
I don't have the ability to test python very well, but my brother does. Just for fun, here is the code that B4J made for the DeepCopy command which is needed for the parts upload:

B4J Code:
Sub DeepCopy(original As Object) As Object
    If original Is Map Then
        Dim originalMap As Map = original
        Dim copyMap As Map
        copyMap.Initialize
        For Each key As Object In originalMap.Keys
            copyMap.Put(key, DeepCopy(originalMap.Get(key)))
        Next
        Return copyMap
    Else If original Is List Then
        Dim originalList As List = original
        Dim copyList As List
        copyList.Initialize
        For Each item As Object In originalList
            copyList.Add(DeepCopy(item))
        Next
        Return copyList
    Else
        ' For primitive types (e.g., numbers, strings), return the item itself
        Return original
    End If
End Sub

We will test it by comparing the output in python and in B4J.
 
Upvote 0

Tim Chapman

Active Member
Licensed User
Longtime User
My current path forward is to get the code converted that is required for creating assistants and creating a thread, etc. so I can upload a file and use it.
Then I will upload all the libraries for B4J so I can use them to make OpenAI make B4J with less/no mistakes. I can do that without Erel giving me anything if he doesn't want to.
Then I will upload the python library and get all the rest of it converted to B4J (and B4A) and clean it up.
Once I have the library working smoothly, I will publish it.
Then I will upload all the libraries for B4A so it will work better.
 
Upvote 0

DonManfred

Expert
Licensed User
Longtime User
My current path forward is to get the code converted that is required for creating assistants and creating a thread, etc. so I can upload a file and use it.
And you have an assistantfile with >512MB?
512MB on CSV-Data is A LOT for an assistant.

Having need for the partsupload is most probably a very rare ussecase.
 
Upvote 0

Tim Chapman

Active Member
Licensed User
Longtime User
After another day of working on this, I have devised a more detailed plan for converting the openai python library to B4J. I will use the API for the library to get me to the entry points in the library for each function. Then the hard part is to gather the source code for every method and class found by drilling down from the entry points. I will use the AI to do a lot of this. Once I have all of the source code for a function, it will convert fairly easily to B4J. Then I will make the B4J library from the results. To test it requires running each function and seeing if I get a valid response from the OpenAI. Such fun!
 
Upvote 0

DonManfred

Expert
Licensed User
Longtime User
I have pdf files that I want to upload that exceed 512 MB.
Ok. Agree, you have a use case :D
I´ve not. That´s why i only use the "standard" fileuploads from OpenAI to feed my Assistant. But that´s not relevant here ;)

Back to you initial Post you wrote
The curl command to upload a part of a file is here:

curl https://api.openai.com/v1/uploads/upload_abc123/parts
-F data="aHR0cHM6Ly9hcGkub3BlbmFpLmNvbS92MS91cGxvYWRz..."
Like Erel already suggest you need to Split the File into smaller parts first.

Based on the Apidescription you need to start with a call to

it returns with a uploadobject

{
"id": "upload_abc123",
"object": "upload",
"bytes": 2147483648,
"created_at": 1719184911,
"filename": "training_examples.jsonl",
"purpose": "fine-tune",
"status": "pending",
"expires_at": 1719127296
}
you then can use the id to upload the parts.
upload the splitted chunks in the rigth order with the partsupload.

That´s how i do see how it should work.
Havn´t tried to use the - for me new methods - regarding UPLOADS so far.
 
Upvote 0

Tim Chapman

Active Member
Licensed User
Longtime User
Ok. Agree, you have a use case :D
I´ve not. That´s why i only use the "standard" fileuploads from OpenAI to feed my Assistant. But that´s not relevant here ;)

Back to you initial Post you wrote

Like Erel already suggest you need to Split the File into smaller parts first.

Based on the Apidescription you need to start with a call to

it returns with a uploadobject


you then can use the id to upload the parts.
upload the splitted chunks in the rigth order with the partsupload.

That´s how i do see how it should work.
Havn´t tried to use the - for me new methods - regarding UPLOADS so far.
Hi Don,
You mention your Assistant. Have you gotten the Create Assistant, Create Thread, Add Message to Thread functions working? I would like to add them to the OpenAI library as well. Any others?
 
Upvote 0

Tim Chapman

Active Member
Licensed User
Longtime User
Converting the python is not working well.
So my code now is below and I am still getting the same error:

"error": {
"message": "'data' is a required property",
"type": "invalid_request_error",
"param": null,
"code": null
}
}


B4J code:
Sub Class_Globals
    Private Root As B4XView ' Root view of the page.
    Private xui As XUI ' Cross-platform UI library.
    Private OpenAIapiKey As String = "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"
    Private Label1 As Label
End Sub

Public Sub Initialize ' Initialization method for the class.
    Button1_Click
End Sub

'This event will be called once, before the page becomes visible.
Private Sub B4XPage_Created (Root1 As B4XView)
    Root = Root1 ' Assign the root view.
    Root.LoadLayout("MainPage") ' Load the layout named "MainPage".
    'Label1.Text = "Please watch the logs.  It takes 5 or 10 minutes to get the final response." & CRLF & "See line 30 and change it to a folder you like and line 36 to a file you like." & CRLF & "Also, see line 70 of the code.  I limited it to 1 part becuse I run out of memory otherwise.   I will fix that after I get an appropriate response from a part upload."
End Sub

'You can see the list of page related events in the B4XPagesManager object. The event name is B4XPage.

Private Sub Button1_Click ' Event handler for Button1 click.
    Private uploadApiUrl As String = "https://api.openai.com/v1/uploads"
    Dim Folder As String = "D:\Downloads"
    
    Log("File.DirApp = " & File.DirApp)
    Log("File.DirAssets = " & File.DirAssets) ' This is the Files Folder.
    Log("File.DirTemp = " & File.DirTemp)
        
    Dim FileName As String = "BOTG_with_licence.pdf"
    Dim Purpose As String = "assistants"
    
    Dim Job As HttpJob
    Job.Initialize("initializeUploadJob", Me)
    Dim FilePath As String = Folder & "\" & FileName
    Job.Tag = FilePath
    Dim fileSize As Long = File.Size(Folder, FileName)
    Dim mimeType As String = GetMimeType1(FileName) ' Adjust MIME type as needed
    Dim json As String = $"{"filename": "${FileName}", "purpose": "${Purpose}", "bytes": ${fileSize}, "mime_type": "${mimeType}"}"$

    Job.PostString(uploadApiUrl, json)
    Job.GetRequest.SetHeader("Authorization", "Bearer " & OpenAIapiKey)
    Job.GetRequest.SetContentType("application/json")
    Job.GetRequest.SetHeader("OpenAI-Beta", "assistants=v2")
    
    Wait For (Job) JobDone (Job As HttpJob) ' Wait for the job to complete.
    
    If Job.Success Then
        LogColor("Job = " & Job,0xFF006D11) ' Log the job
        Log("Initialize Upload Job Response = " & Job.GetString) ' Log the job result.
    
        ' Convert JSON string to map
        Dim parser As JSONParser
        parser.Initialize(Job.GetString)
        Dim result As Map = parser.NextObject
    
        Dim FileID As String = result.Get("id")
        Log("File ID = " & FileID)

        ' Proceed to upload parts
        Dim fileSize As Long = File.Size(Folder, FileName)
        Log("File Size = " & fileSize)
        Dim partSize As Long = 64 * 1024 * 1024 ' 64 MB
        Dim partCount As Int = Ceil(fileSize / partSize)
        Log("PartCount = " & partCount)
        For i = 0 To 1 'partCount - 1
            Dim start As Long = i * partSize
            Dim end1 As Long = Min((i + 1) * partSize, fileSize)
            'Log("Uploadpart called: " & start & "," & end1 & "," & Folder & "," &  FileName & "," &  FileID & "," &  apiKey)
            LogColor("***",0xFFFF3700)

            ' *** THE CODE WORKS PROPERLY UP TO HERE ***

            'Upload Part
            Dim partJob As HttpJob
            partJob.Initialize("uploadPartJob", Me)
    
            Dim url As String = "https://api.openai.com/v1/uploads/" & FileID & "/parts"
    
            Dim length As Long = end1 - start
            Dim raf As RandomAccessFile
            raf.Initialize(Folder, FileName, False)
            Log("Uploading " & length & " bytes")
            Dim partData(length) As Byte
            raf.ReadBytes(partData, 0, length, start)
            raf.Close
    
            'Start Here
            ' Create a stream to build the multipart request body
            Dim stream As OutputStream
            stream.InitializeToBytesArray(0)

            ' Define boundary
            Dim boundary As String = "---------------------------1461124740692"
            Dim eol As String = Chr(13) & Chr(10)

            ' Add file data
            Dim fileHeader As String = _
    $"--${boundary}
Content-Disposition: form-data; name="data"; filename="${FileName}"
Content-Type: ${mimeType}

"$
            stream.WriteBytes(fileHeader.Replace(CRLF, eol).GetBytes("UTF8"), 0, fileHeader.Length)
            stream.WriteBytes(partData, 0, length)

            ' End of multipart
            Dim endBoundary As String = eol & "--" & boundary & "--" & eol
            stream.WriteBytes(endBoundary.GetBytes("UTF8"), 0, endBoundary.Length)

            partJob.PostBytes(url, stream.ToBytesArray)
            partJob.GetRequest.SetHeader("Authorization", "Bearer " & OpenAIapiKey)
            partJob.GetRequest.SetContentType("multipart/form-data; boundary=" & boundary)
            'LogColor("Part Job = " & partJob, 0xFF006D11)
            Wait For (partJob) JobDone (partJob As HttpJob)

            If partJob.Success Then
                LogColor("PartUpload Job Response = " & partJob.GetString, 0xFFFF3700)
            Else
                xui.MsgboxAsync(partJob.JobName & ": " & partJob.ErrorMessage, "Error")
            End If
            partJob.Release
        Next
    End If
End Sub

Private Sub GetMimeType1(fileName As String) As String
    Dim extension As String = fileName.SubString(fileName.LastIndexOf(".") + 1).ToLowerCase
    Select extension
        Case "txt"
            Return "text/plain"
        Case "html", "htm"
            Return "text/html"
        Case "jpg", "jpeg"
            Return "image/jpeg"
        Case "png"
            Return "image/png"
        Case "pdf"
            Return "application/pdf"
        Case "doc"
            Return "application/msword"
        Case "docx"
            Return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
        Case "xls"
            Return "application/vnd.ms-excel"
        Case "xlsx"
            Return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
        Case Else
            Return "application/octet-stream"
    End Select
End Sub
 
Upvote 0

drgottjr

Expert
Licensed User
Longtime User
well, i'm very close. a tiny problem at the end. late now. i'll pick it up in the morning

B4X:
filesize: 10956291
sending: {"filename":"uploadtest2.csv","purpose":"assistants","mime_type":"text\/csv","bytes":10956291}
Call B4XPages.GetManager.LogEvents = True to enable logging B4XPages events.
OpenAIUploadID = upload_yOQKlpuXybbdkYq9ktnSVUZp
uploading chunks
url now: https://api.openai.com/v1/uploads/upload_yOQKlpuXybbdkYq9ktnSVUZp/parts
total bytes to transfer: 10956291
processing chunkcount 1
read 750000 bytes
uploading temp1
{
  "id": "part_FMhppgOPp821l6FuucnRxOgl",
  "object": "upload.part",
  "created_at": 1726280591,
  "upload_id": "upload_yOQKlpuXybbdkYq9ktnSVUZp"
}
job id: part_FMhppgOPp821l6FuucnRxOgl
bytes so far: 750000 out of 10956291 bytes
back for more
processing chunkcount 2
read 750000 bytes
uploading temp2
{
  "id": "part_7gokp4BUUjJHCfRJ5FDH3DiO",
  "object": "upload.part",
  "created_at": 1726280594,
  "upload_id": "upload_yOQKlpuXybbdkYq9ktnSVUZp"
}
job id: part_7gokp4BUUjJHCfRJ5FDH3DiO
bytes so far: 1500000 out of 10956291 bytes
back for more
processing chunkcount 3
read 750000 bytes
uploading temp3
{
  "id": "part_qK7GR9WvFoIit1PkW4JPoyNO",
  "object": "upload.part",
  "created_at": 1726280596,
  "upload_id": "upload_yOQKlpuXybbdkYq9ktnSVUZp"
}
job id: part_qK7GR9WvFoIit1PkW4JPoyNO
bytes so far: 2250000 out of 10956291 bytes
back for more
processing chunkcount 4
read 750000 bytes
uploading temp4
{
  "id": "part_KgfrF8xeVkUXju9oF4UrNbVu",
  "object": "upload.part",
  "created_at": 1726280599,
  "upload_id": "upload_yOQKlpuXybbdkYq9ktnSVUZp"
}
job id: part_KgfrF8xeVkUXju9oF4UrNbVu
bytes so far: 3000000 out of 10956291 bytes
back for more
processing chunkcount 5
read 750000 bytes
uploading temp5
{
  "id": "part_KbGRYQ0XDcyb1hP8R3VYfEhp",
  "object": "upload.part",
  "created_at": 1726280602,
  "upload_id": "upload_yOQKlpuXybbdkYq9ktnSVUZp"
}
job id: part_KbGRYQ0XDcyb1hP8R3VYfEhp
bytes so far: 3750000 out of 10956291 bytes
back for more
processing chunkcount 6
read 750000 bytes
uploading temp6
{
  "id": "part_HQ61IolIBSegIci37lrs5LOR",
  "object": "upload.part",
  "created_at": 1726280604,
  "upload_id": "upload_yOQKlpuXybbdkYq9ktnSVUZp"
}
job id: part_HQ61IolIBSegIci37lrs5LOR
bytes so far: 4500000 out of 10956291 bytes
back for more
processing chunkcount 7
read 750000 bytes
uploading temp7
{
  "id": "part_bReDfFLMsRwEeSuL2mjJ3zlS",
  "object": "upload.part",
  "created_at": 1726280607,
  "upload_id": "upload_yOQKlpuXybbdkYq9ktnSVUZp"
}
job id: part_bReDfFLMsRwEeSuL2mjJ3zlS
bytes so far: 5250000 out of 10956291 bytes
back for more
processing chunkcount 8
read 750000 bytes
uploading temp8
{
  "id": "part_8v45r5q3KGYWUcTmHm9XwbzJ",
  "object": "upload.part",
  "created_at": 1726280609,
  "upload_id": "upload_yOQKlpuXybbdkYq9ktnSVUZp"
}
job id: part_8v45r5q3KGYWUcTmHm9XwbzJ
bytes so far: 6000000 out of 10956291 bytes
back for more
processing chunkcount 9
read 750000 bytes
uploading temp9
{
  "id": "part_VLS7q3NoROEomeXfGFFNLKyV",
  "object": "upload.part",
  "created_at": 1726280612,
  "upload_id": "upload_yOQKlpuXybbdkYq9ktnSVUZp"
}
job id: part_VLS7q3NoROEomeXfGFFNLKyV
bytes so far: 6750000 out of 10956291 bytes
back for more
processing chunkcount 10
read 750000 bytes
uploading temp10
{
  "id": "part_Bvi1wuTkjhxIzetI5cYYI2IA",
  "object": "upload.part",
  "created_at": 1726280614,
  "upload_id": "upload_yOQKlpuXybbdkYq9ktnSVUZp"
}
job id: part_Bvi1wuTkjhxIzetI5cYYI2IA
bytes so far: 7500000 out of 10956291 bytes
back for more
processing chunkcount 11
read 750000 bytes
uploading temp11
{
  "id": "part_F07pQ5HOYFyy1ZZRHJ8yUzsC",
  "object": "upload.part",
  "created_at": 1726280617,
  "upload_id": "upload_yOQKlpuXybbdkYq9ktnSVUZp"
}
job id: part_F07pQ5HOYFyy1ZZRHJ8yUzsC
bytes so far: 8250000 out of 10956291 bytes
back for more
processing chunkcount 12
read 750000 bytes
uploading temp12
{
  "id": "part_goCujT6vhH7USkJWWNvR9JDn",
  "object": "upload.part",
  "created_at": 1726280620,
  "upload_id": "upload_yOQKlpuXybbdkYq9ktnSVUZp"
}
job id: part_goCujT6vhH7USkJWWNvR9JDn
bytes so far: 9000000 out of 10956291 bytes
back for more
processing chunkcount 13
read 750000 bytes
uploading temp13
{
  "id": "part_YCIpWm6D1yOHSNO1lo19xlF7",
  "object": "upload.part",
  "created_at": 1726280622,
  "upload_id": "upload_yOQKlpuXybbdkYq9ktnSVUZp"
}
job id: part_YCIpWm6D1yOHSNO1lo19xlF7
bytes so far: 9750000 out of 10956291 bytes
back for more
processing chunkcount 14
read 750000 bytes
uploading temp14
{
  "id": "part_d3umCGMSUPAV4hp7ck4Mj080",
  "object": "upload.part",
  "created_at": 1726280625,
  "upload_id": "upload_yOQKlpuXybbdkYq9ktnSVUZp"
}
job id: part_d3umCGMSUPAV4hp7ck4Mj080
bytes so far: 10500000 out of 10956291 bytes
back for more
processing chunkcount 15
read 456291 bytes
uploading temp15
ResponseError. Reason: Bad Request, Response: {
  "error": {
    "message": "Uploaded bytes will exceed the specified file size of 10956291. Uploaded bytes: 10500000, current part size: 750000, specified file size: 10956291.",
    "type": "invalid_request_error",
    "param": "uploaded_bytes",
    "code": "uploaded_bytes_above_specified"
  }
}
{
  "error": {
    "message": "Uploaded bytes will exceed the specified file size of 10956291. Uploaded bytes: 10500000, current part size: 750000, specified file size: 10956291.",
    "type": "invalid_request_error",
    "param": "uploaded_bytes",
    "code": "uploaded_bytes_above_specified"
  }
}
bytes so far: 10956291 out of 10956291 bytes
back for more
completing upload
ResponseError. Reason: Bad Request, Response: {
  "error": {
    "message": "We were unable to complete your upload because the uploaded bytes do not match the specified bytes of 10956291. Please exclude part_ids or start a new upload.",
    "type": "invalid_request_error",
    "param": null,
    "code": "file_size_mismatch"
  }
}

what i did:
i took a big file (10MB)
initially i just uploaded the whole thing at once to see if i followed the api.
then i broke it up into chunks. for the purpose of testing, i set the chunksize at 750000 bytes to keep the number of chunks low and upload times short.
as you can see from the log, it works nice. i'm not sure yet what the issue is at the end when i go to complete the upload, but i get an error. oddly, a quick read of the log shows that i needed to upload 10956291 bytes, and that's what i seemed to do. but i'll deal with it tomorrow.

by the way, you don't need randomaccess to read a file in chunks. also, i probably abused your key with my testing. openai seems to apply its own slowdown; if i tried and found an error, fixed it and went back to try again, openai basically wouldn't answer. if i had a piece of toast and came back a while later, then openai was happy. sorry.
 
Upvote 0

Tim Chapman

Active Member
Licensed User
Longtime User
I got it working. Code is below.

Add your API key, folder and file to upload on lines 4, 23 and 24.

I will have this in the OpenAI library soon.

Thank you everyone for the help!

B4J upload file in parts to OpenAI:
Sub Class_Globals
    Private Root As B4XView ' Root view of the page.
    Private xui As XUI ' Cross-platform UI library.
    Private OpenAIapiKey As String = "YOUR_API_KEY"
    Private Label1 As Label
End Sub

Public Sub Initialize ' Initialization method for the class.
    Button1_Click
End Sub

'This event will be called once, before the page becomes visible.
Private Sub B4XPage_Created (Root1 As B4XView)
    Root = Root1 ' Assign the root view.
    Root.LoadLayout("MainPage") ' Load the layout named "MainPage".
    'Label1.Text = "Please watch the logs.  It takes 5 or 10 minutes to get the final response." & CRLF & "See line 30 and change it to a folder you like and line 36 to a file you like." & CRLF & "Also, see line 70 of the code.  I limited it to 1 part becuse I run out of memory otherwise.   I will fix that after I get an appropriate response from a part upload."
End Sub

'You can see the list of page related events in the B4XPagesManager object. The event name is B4XPage.

Private Sub Button1_Click
    Private uploadApiUrl As String = "https://api.openai.com/v1/uploads"
    Dim Folder As String = "D:\Downloads"
    Dim FileName As String = "BOTG_with_licence.pdf"
    Dim Purpose As String = "fine-tune" ' Adjust purpose as needed
    Dim FilePath As String = Folder & "\" & FileName
    Dim fileSize As Long = File.Size(Folder, FileName)
    Dim mimeType As String = GetMimeType1(FileName) ' Adjust MIME type as needed

    ' Initialize the upload
    Dim Job As HttpJob
    Job.Initialize("initializeUploadJob", Me)
    Dim json As String = $"{"filename": "${FileName}", "purpose": "${Purpose}", "bytes": ${fileSize}, "mime_type": "${mimeType}"}"$
    Job.PostString(uploadApiUrl, json)
    Job.GetRequest.SetHeader("Authorization", "Bearer " & OpenAIapiKey)
    Job.GetRequest.SetContentType("application/json")
    Job.GetRequest.SetHeader("OpenAI-Beta", "assistants=v2")
   
    Wait For (Job) JobDone (Job As HttpJob)
   
    If Job.Success Then
        Log("Initialize Upload Job Response = " & Job.GetString)
        Dim parser As JSONParser
        parser.Initialize(Job.GetString)
        Dim result As Map = parser.NextObject
        Dim FileID As String = result.Get("id")
        Log("File ID = " & FileID)

        ' Proceed to upload parts
        Dim partSize As Long = 64 * 1024 * 1024 ' 64 MB
        Dim partCount As Int = Ceil(fileSize / partSize)
        Log("PartCount = " & partCount)
        Dim partIds As List
        partIds.Initialize
       
        For i = 0 To partCount - 1
            Dim start As Long = i * partSize
            Dim end1 As Long = Min((i + 1) * partSize, fileSize)
            Dim length As Long = end1 - start
            Log("Uploading part " & (i + 1) & " of " & partCount & ", bytes " & start & " to " & end1)
           
            ' Read the part data
            Dim raf As RandomAccessFile
            raf.Initialize(Folder, FileName, False)
            Dim partData(length) As Byte
            raf.ReadBytes(partData, 0, length, start)
            raf.Close
           
            ' Build the multipart form-data body
            Dim stream As OutputStream
            stream.InitializeToBytesArray(0)
           
            ' Define boundary
            Dim boundary As String = "---------------------------1461124740692"
            Dim eol As String = Chr(13) & Chr(10)
           
            ' Build the header
            Dim fileHeader As String = _
$"--${boundary}
Content-Disposition: form-data; name="data"; filename="${FileName}"
Content-Type: ${mimeType}

"$
            fileHeader = fileHeader.Replace(CRLF, eol)
            Dim headerBytes() As Byte = fileHeader.GetBytes("UTF8")
            stream.WriteBytes(headerBytes, 0, headerBytes.Length)
           
            ' Write the data
            stream.WriteBytes(partData, 0, partData.Length)
           
            ' Write the end boundary
            Dim endBoundary As String = eol & "--" & boundary & "--" & eol
            Dim endBoundaryBytes() As Byte = endBoundary.GetBytes("UTF8")
            stream.WriteBytes(endBoundaryBytes, 0, endBoundaryBytes.Length)
           
            ' Initialize and send the job
            Dim partJob As HttpJob
            Dim ix As Int = i + 1
            partJob.Tag = "File # "& ix & " of " & partCount
            partJob.Initialize("uploadPartJob", Me)
            Dim url As String = "https://api.openai.com/v1/uploads/" & FileID & "/parts"
            partJob.PostBytes(url, stream.ToBytesArray)
            partJob.GetRequest.SetHeader("Authorization", "Bearer " & OpenAIapiKey)
            partJob.GetRequest.SetContentType("multipart/form-data; boundary=" & boundary)

            ' Log the Request Details
            Log("Uploading to URL: " & url)
            Log("Authorization: Bearer " & OpenAIapiKey)
            Log("Content-Type: multipart/form-data; boundary=" & boundary)
            ' Log partial stream content, first 256 characters, for inspection
            'Dim streamContent As String = BytesToString(stream.ToBytesArray, 0, Min(256, stream.ToBytesArray.Length), "UTF8")
            'Log("Request Body Content (partial): " & streamContent)
           
            Wait For (partJob) JobDone (partJob As HttpJob)
           
            If partJob.Success Then
                ' Parse the response
                Dim parser2 As JSONParser
                parser2.Initialize(partJob.GetString)
                Dim result2 As Map = parser2.NextObject
                Dim partId As String = result2.Get("id")
                partIds.Add(partId)
                Log("Uploaded part " & (i + 1) & " with id " & partId)
            Else
                Log("Error uploading part " & (i + 1) & ": " & partJob.ErrorMessage)
                ' Handle the error appropriately
                Return ' Exit the sub if there's an error
            End If
           
            partJob.Release
        Next
       
        ' Complete the upload
        Dim partIdsJson As String = "["
        For i = 0 To partIds.Size - 1
            Dim partId As String = partIds.Get(i)
            partIdsJson = partIdsJson & $" "${partId}" "$
            If i < partIds.Size - 1 Then
                partIdsJson = partIdsJson & ","
            End If
        Next
        partIdsJson = partIdsJson & "]"
       
        ' Build the complete request JSON
        Dim completeJson As String = $"{"part_ids": ${partIdsJson}}"$

        ' Send the complete request
        Dim completeJob As HttpJob
        completeJob.Initialize("completeUploadJob", Me)
        Dim completeUrl As String = "https://api.openai.com/v1/uploads/" & FileID & "/complete"
        completeJob.PostString(completeUrl, completeJson)
        completeJob.GetRequest.SetHeader("Authorization", "Bearer " & OpenAIapiKey)
        completeJob.GetRequest.SetContentType("application/json")

        Wait For (completeJob) JobDone (completeJob As HttpJob)

        If completeJob.Success Then
            Log("Upload completed successfully.")
            Log("Response: " & completeJob.GetString)
        Else
            Log("Error completing upload: " & completeJob.ErrorMessage)
        End If

        completeJob.Release
    Else
        Log("Error initializing upload: " & Job.ErrorMessage)
    End If
    Job.Release
End Sub

Private Sub GetMimeType1(fileName As String) As String
    Dim extension As String = fileName.SubString(fileName.LastIndexOf(".") + 1).ToLowerCase
    Select extension
        Case "txt"
            Return "text/plain"
        Case "html", "htm"
            Return "text/html"
        Case "jpg", "jpeg"
            Return "image/jpeg"
        Case "png"
            Return "image/png"
        Case "pdf"
            Return "application/pdf"
        Case "doc"
            Return "application/msword"
        Case "docx"
            Return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
        Case "xls"
            Return "application/vnd.ms-excel"
        Case "xlsx"
            Return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
        Case Else
            Return "application/octet-stream"
    End Select
End Sub
 
Upvote 0
Solution

Tim Chapman

Active Member
Licensed User
Longtime User
I would love to collaborate on this project if anyone is willing. Please PM me.

I currently have the following functions working:
Chat
TTS
Image response
Create Assistant
Create Thread
File Upload
File Upload in parts
Delete file
Cancel upload
List files

I have built the user interface for all that, but it is not complete yet.
 
Last edited:
Upvote 0
Top