diff --git a/dfsclient/src/main/java/org/hpccsystems/dfs/client/BinaryRecordWriter.java b/dfsclient/src/main/java/org/hpccsystems/dfs/client/BinaryRecordWriter.java index 00e362399..601aefb11 100644 --- a/dfsclient/src/main/java/org/hpccsystems/dfs/client/BinaryRecordWriter.java +++ b/dfsclient/src/main/java/org/hpccsystems/dfs/client/BinaryRecordWriter.java @@ -475,10 +475,13 @@ else if (fd.getDataLen() == 8) case STRING: { String value = fieldValue != null ? (String) fieldValue : ""; - int eosIdx = value.indexOf('\0'); - if (eosIdx > -1) + if (fd.getFieldType() == FieldType.VAR_STRING) { - value = value.substring(0,eosIdx); + int eosIdx = value.indexOf('\0'); + if (eosIdx > -1) + { + value = value.substring(0,eosIdx); + } } byte[] data = new byte[0]; @@ -609,10 +612,23 @@ else if (fd.getSourceType() == HpccSrcType.QSTRING) if (fd.getFieldType() == FieldType.VAR_STRING) { byte nullByte = '\0'; - this.buffer.put(nullByte); if (fd.getSourceType().isUTF16()) { - this.buffer.put(nullByte); + boolean needsNullAdded = data.length < 2 + || data[data.length - 1] != nullByte + || data[data.length - 2] != nullByte; + if (needsNullAdded) { + this.buffer.put(nullByte); + this.buffer.put(nullByte); + } + } + else + { + boolean needsNullAdded = data.length < 1 + || data[data.length - 1] != nullByte; + if (needsNullAdded) { + this.buffer.put(nullByte); + } } } } diff --git a/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSReadWriteTest.java b/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSReadWriteTest.java index 6d2720405..c2d306cde 100644 --- a/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSReadWriteTest.java +++ b/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSReadWriteTest.java @@ -88,6 +88,93 @@ public void readWithForcedTimeoutTest() throws Exception assertEquals("Not all records loaded",expectedCounts[0], records.size()); } + @Test + public void allCharsNullTest() throws Exception + { + // Unicode + { + FieldDef recordDef = null; + { + FieldDef[] fieldDefs = new FieldDef[2]; + fieldDefs[0] = new FieldDef("uni", FieldType.STRING, "STRING", 100, false, false, HpccSrcType.UTF16LE, new FieldDef[0]); + fieldDefs[1] = new FieldDef("fixedUni", FieldType.STRING, "STRING", 100, true, false, HpccSrcType.UTF16LE, new FieldDef[0]); + + recordDef = new FieldDef("RootRecord", FieldType.RECORD, "rec", 4, false, false, HpccSrcType.LITTLE_ENDIAN, fieldDefs); + } + + List records = new ArrayList(); + int maxUTF16BMPChar = Character.MAX_CODE_POINT; + for (int i = 0; i < maxUTF16BMPChar; i++) { + String strMidEOS = ""; + for (int j = 0; j < 98; j++, i++) { + if (j == 50) { + strMidEOS += "\0"; + } + strMidEOS += Character.toString((char) i); + } + + Object[] fields = {strMidEOS, strMidEOS}; + records.add(new HPCCRecord(fields, recordDef)); + } + + String fileName = "unicode::all_chars::test"; + writeFile(records, fileName, recordDef, connTO); + + HPCCFile file = new HPCCFile(fileName, connString , hpccUser, hpccPass); + List readRecords = readFile(file, 10000, false, false, BinaryRecordReader.TRIM_STRINGS); + + for (int i = 0; i < records.size(); i++) { + HPCCRecord record = records.get(i); + HPCCRecord readRecord = readRecords.get(i); + if (readRecord.equals(record) == false) + { + System.out.println("Record: " + i + " did not match\n" + record + "\n" + readRecord); + } + } + } + + // SBC / ASCII + { + FieldDef recordDef = null; + { + FieldDef[] fieldDefs = new FieldDef[2]; + fieldDefs[0] = new FieldDef("str", FieldType.STRING, "STRING", 10, false, false, HpccSrcType.SINGLE_BYTE_CHAR, new FieldDef[0]); + fieldDefs[1] = new FieldDef("fixedStr", FieldType.STRING, "STRING", 10, true, false, HpccSrcType.SINGLE_BYTE_CHAR, new FieldDef[0]); + + recordDef = new FieldDef("RootRecord", FieldType.RECORD, "rec", 4, false, false, HpccSrcType.LITTLE_ENDIAN, fieldDefs); + } + + List records = new ArrayList(); + for (int i = 0; i < 255; i++) { + String strMidEOS = ""; + for (int j = 0; j < 9; j++, i++) { + if (j == 5) { + strMidEOS += "\0"; + } + strMidEOS += Character.toString((char) i); + } + + Object[] fields = {strMidEOS, strMidEOS}; + records.add(new HPCCRecord(fields, recordDef)); + } + + String fileName = "ascii::all_chars::test"; + writeFile(records, fileName, recordDef, connTO); + + HPCCFile file = new HPCCFile(fileName, connString , hpccUser, hpccPass); + List readRecords = readFile(file, 10000, false, false, BinaryRecordReader.TRIM_STRINGS); + + for (int i = 0; i < records.size(); i++) { + HPCCRecord record = records.get(i); + HPCCRecord readRecord = readRecords.get(i); + if (readRecord.equals(record) == false) + { + System.out.println("Record: " + i + " did not match\n" + record + "\n" + readRecord); + } + } + } + } + @Test public void integrationReadWriteBackTest() throws Exception {