Byte overflow in BinaryRecordReader STRINGXX fields containing unicode

When reading data from hpcc and writing it to oracle I found that some fields containing odd characters had more bytes than the string field allowed.

STRING120 field company_name was returning strings that were 121 or 122 bytes long for field values like

'2005 MAXWELL LANE CONDOMINIUMS OWNERSÆ ASSOCIATION '

'RUBY�S BATH BOUTIQUE, INC. '

'VICTORIA�S SECRET STORES, LLC '

 

You can recreate this issue with the following code:

`````

 

@Test
public void byteWierdness() throws Exception

{ connection = new Connection("http://alpha_vault_thor_esp.risk.regn.net:8010/"); connection.setCredentials("dleed","passwd"); connection.setConnectTimeoutMilli(1000000); connection.setSocketTimeoutMilli(1000000); platform = Platform.get(connection); HPCCFile file = new HPCCFile("~drea::testunicode", "http://alpha_vault_thor_esp.risk.regn.net:8010/" ,"dleed","passwd"); checkFile(file, 1000000); }

public void checkFile(HPCCFile file, Integer connectTimeoutMillis) throws Exception
{
if (file == null)

{ Assert.fail("HPCCFile construction failed."); }

if (connectTimeoutMillis != null)

{ file.setFileAccessExpirySecs(connectTimeoutMillis/1000); }

DataPartition[] fileParts = file.getFileParts();
if (fileParts == null || fileParts.length == 0)

{ Assert.fail("No file parts found"); }

FieldDef originalRD = file.getRecordDefinition();
if (originalRD == null || originalRD.getNumDefs() == 0)

{ Assert.fail("Invalid or null record definition"); }

ArrayList<HpccRemoteFileReader<HPCCRecord>> fileReaders = new ArrayList<HpccRemoteFileReader<HPCCRecord>>();
for (int i = 0; i < fileParts.length; i++)
{
try

{ HPCCRecordBuilder recordBuilder = new HPCCRecordBuilder(file.getProjectedRecordDefinition()); HpccRemoteFileReader<HPCCRecord> fileReader = new HpccRemoteFileReader<HPCCRecord>(fileParts[i], originalRD, recordBuilder); fileReaders.add(fileReader); }

catch (Exception e)
{
Throwable cause = e.getCause();
if (cause instanceof HpccFileException)

{ if (cause.getCause() instanceof java.net.SocketTimeoutException) System.out.println("File Read failed due to connect timeout, if filepart location is on an unreachable ip consider setting up a cluster remapper"); }

Assert.fail("Error constructing reader: " + e.getMessage());
}
}
for (int i = 0; i < fileReaders.size(); i++)
{
HpccRemoteFileReader<HPCCRecord> fileReader = fileReaders.get;
while (fileReader.hasNext())
{
HPCCRecord record = fileReader.next();
String companyname=(String)record.getField(1);
if (companyname != null && companyname.getBytes().length>120)

{ Assert.fail("Company name '" + companyname + "' is longer than 120 bytes:" + companyname.getBytes().length); }

}
fileReader.close();

}
}

`````