I spoke too soon as there was a major bug in the code to determine any separator Char.
Hadn't tested fully, that is not tested on enclosed separators and also not tested on files with no separators, so where there is only one column.
Also in the code to determine the separator char there could be less lines in the file than the specified number of lines to test and that needed fixing.
Have fixed this now, but won't close this thread for now.
Adjusted code:
Code:
Sub CountFieldsInTextLine(strCSVLine As String, strSeparator As String, strEncloser As String) As Int
Dim c As Int
Dim iEndIndex As Int
Dim bInsideQuotes As Boolean = False
If strSeparator.Length = 0 Then
Return 1
End If
For iEndIndex = 0 To strCSVLine.Length - 1
If strCSVLine.CharAt(iEndIndex) = strEncloser Then
bInsideQuotes = bInsideQuotes = False
Else
If strCSVLine.CharAt(iEndIndex) = strSeparator Then
If bInsideQuotes = False Then
c = c + 1
End If
End If
End If
Next
Return c + 1
End Sub
Sub GetCharCounts(strLine As String, strEncloser As String) As Map
Dim i As Int
Dim bInsideQuotes As Boolean
Dim mapCountChars As Map
Dim iCount As Int
Dim oChar As Char
mapCountChars.Initialize
For i = 0 To strLine.Length - 1
oChar = strLine.CharAt(i)
If oChar = strEncloser Then
bInsideQuotes = bInsideQuotes = False
Else
If bInsideQuotes = False Then
If mapCountChars.ContainsKey(oChar) Then
iCount = mapCountChars.Get(oChar)
mapCountChars.Put(oChar, iCount + 1)
Else
mapCountChars.Put(oChar, 1)
End If
End If
End If
Next
Return mapCountChars
End Sub
Sub GetSeparatorCharFromFile(strFolder As String, strFile As String, _
strCharSet As String, strEncloser As String, iLines As Int) As ResumableSub
Dim i As Int
Dim n As Int
Dim oTR As TextReader
Dim strLine As String
Dim iCount As Int
Dim lstMaps As List
Dim oMap As Map
Dim oMapCharAndCount As Map
Dim strKey As String
Dim strPossibleSeparator As Char
Dim strExclude As String = " ' "" 0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" 'includes quote and double-quote
oTR.Initialize2(File.OpenInput(strFolder, strFile), strCharSet)
lstMaps.Initialize
Do While True
strLine = oTR.ReadLine
If strLine = Null Then
iLines = n
Exit
End If
lstMaps.Add(GetCharCounts(strLine, strEncloser))
n = n + 1
If n = iLines Then
Exit
End If
Loop
oTR.Close
oMapCharAndCount.Initialize
For i = 0 To lstMaps.Size - 1
oMap = lstMaps.Get(i)
For Each oKey As Object In oMap.Keys
strKey = oKey & oMap.Get(oKey)
If oMapCharAndCount.ContainsKey(strKey) Then
iCount = oMapCharAndCount.Get(strKey)
oMapCharAndCount.Put(strKey, iCount + 1)
Else
oMapCharAndCount.Put(strKey, 1)
End If
Next
Next
For Each oKey As Object In oMapCharAndCount.Keys
'Log(oKey)
If iLines = oMapCharAndCount.Get(oKey) Then
strPossibleSeparator = CStr(oKey).CharAt(0)
'Log("strPossibleSeparator: |" & strPossibleSeparator & "|")
If strExclude.Contains(strPossibleSeparator) = False Then
Return strPossibleSeparator
End If
End If
Next
'need to return something and can't return Null
Return ""
End Sub
Sub ParseCSVLine(strCSVLine As String, iCols As Int, strSeparator As String, strEncloser As String) As String()
'Log("ParseCSVLine, iCols: "& iCols)
Dim c As Int
Dim arrValues(iCols) As String
Dim iStartIndex As Int
Dim iEndIndex As Int
Dim bInsideQuotes As Boolean = False
If strSeparator.Length = 0 Then
arrValues(0) = strCSVLine
Return arrValues
End If
For iEndIndex = 0 To strCSVLine.Length - 1
If strCSVLine.CharAt(iEndIndex) = strEncloser Then
bInsideQuotes = bInsideQuotes = False
Else
If strCSVLine.CharAt(iEndIndex) = strSeparator Then
If bInsideQuotes = False Then
arrValues(c) = strCSVLine.SubString2(iStartIndex, iEndIndex)
iStartIndex = iEndIndex + 1
c = c + 1
End If
End If
End If
Next
arrValues(c) = strCSVLine.SubString2(iStartIndex, iEndIndex )
Return arrValues
End Sub
Also I am not sure if it is OK to use TextReader.
Have seen posts where it is not recommended to use this, unless you are dealing with files that are not UTF-8 encoded.
I wonder if this should be stated as:
unless you are dealing with files that are not UTF-8 encoded or files that are very large.
Also I am interested to know what exactly is wrong with using TextReader.
RBS
