Changeset 4122

Timestamp:

2010-07-07 14:35:45 (3 years ago)

Author:

bruno

Message:

AvroConverter?: when deserializing records, do not rely on the TypeManager? to determine the primitive value type of fields, but rather transport this info as part of the Avro message. Otherwise:

• for each field another remote call is done to retrieve its field type
• this could be solved if in the future we would have a field type cache in the client, but right now it is not clear if we will ever have such cache
• since the name of fields can change, the type of the transported field might not correspond to the field type loaded (from cache or direct). However, this issue is still present, since it know simply moves to the server side. Anyway, such name changes should be rare.

Location:

projects/lily/trunk/repository/impl/src

Files:

• main/avro/lily.avpr (modified) (2 diffs)
• main/java/org/lilycms/repository/avro/AvroConverter.java (modified) (4 diffs)
• test/java/org/lilycms/repository/impl/test/AvroConverterTest.java (modified) (1 diff)

projects/lily/trunk/repository/impl/src/main/avro/lily.avpr

@@ -10 +10 @@
         {"name": "namespace", "type": ["string", "null"]},
         {"name": "name", "type": "string"}
+      ]
+    },
+
+    {
+      "name": "AvroField",
+      "type": "record",
+      "fields": [
+        {"name": "name", "type": "string"},
+        {"name": "primitiveType", "type": "string"},
+        {"name": "multiValue", "type": "boolean"},
+        {"name": "hierarchical", "type": "boolean"},
+        {"name": "value", "type": "bytes"}
       ]
     },
@@ -23 +35 @@
         {"name": "scopeRecordTypeIds", "type": {"type": "map", "values": "string"}},
         {"name": "scopeRecordTypeVersions", "type": {"type": "map", "values": "long"}},
-        {"name": "fields", "type": {"type": "map", "values": "bytes"}},
+        {"name": "fields", "type": {"type": "array", "items": "AvroField"}},
         {"name": "fieldsToDelete", "type": {"type": "array", "items": "string"}}
       ]

projects/lily/trunk/repository/impl/src/main/java/org/lilycms/repository/avro/AvroConverter.java

@@ -54 +54 @@
             }
         }
+
         // Fields
-        Map<Utf8, ByteBuffer> fields = avroRecord.fields;
-        if (fields != null) {
-            for (Entry<Utf8, ByteBuffer> field : fields.entrySet()) {
-                QName name = decodeQName(convert(field.getKey()));
-                Object value = typeManager.getFieldTypeByName(name).getValueType().fromBytes(field.getValue().array());
+        if (avroRecord.fields != null) {
+            for (AvroField field : avroRecord.fields) {
+                QName name = decodeQName(convert(field.name));
+                ValueType valueType = typeManager.getValueType(convert(field.primitiveType), field.multiValue, field.hierarchical);
+                Object value = valueType.fromBytes(field.value.array());
                 record.setField(name, value);
             }
         }
+
         // FieldsToDelete
         GenericArray<Utf8> avroFieldsToDelete = avroRecord.fieldsToDelete;
@@ -104 +106 @@
             }
         }
+
         // Fields
-        avroRecord.fields = new HashMap<Utf8, ByteBuffer>();
+        avroRecord.fields = new GenericData.Array<AvroField>(record.getFields().size(), Schema.createArray(AvroField.SCHEMA$));
         for (Entry<QName, Object> field : record.getFields().entrySet()) {
-            QName name = field.getKey();
+            AvroField avroField = new AvroField();
+            avroField.name = convert(encodeQName(field.getKey()));
+
             FieldType fieldType;
             try {
-                fieldType = typeManager.getFieldTypeByName(name);
+                fieldType = typeManager.getFieldTypeByName(field.getKey());
             } catch (FieldTypeNotFoundException e) {
                 throw convert(e);
@@ -116 +121 @@
                 throw convert(e);
             }
+
+            avroField.primitiveType = convert(fieldType.getValueType().getPrimitive().getName());
+            avroField.multiValue = fieldType.getValueType().isMultiValue();
+            avroField.hierarchical = fieldType.getValueType().isHierarchical();
+
             byte[] value = fieldType.getValueType().toBytes(field.getValue());
             ByteBuffer byteBuffer = ByteBuffer.allocate(value.length);
@@ -121 +131 @@
             byteBuffer.put(value);
             byteBuffer.reset();
-            avroRecord.fields.put(new Utf8(encodeQName(name)), byteBuffer);
-        }
+
+            avroField.value = byteBuffer;
+
+            avroRecord.fields.add(avroField);
+        }
+
         // FieldsToDelete
         List<QName> fieldsToDelete = record.getFieldsToDelete();

projects/lily/trunk/repository/impl/src/test/java/org/lilycms/repository/impl/test/AvroConverterTest.java

@@ -346 +346 @@
         fieldType.getValueType();
         expectLastCall().andReturn(valueType).anyTimes();
-        control.replay();
+        typeManager.getValueType("STRING", false, false);
+        expectLastCall().andReturn(valueType).anyTimes();
+        control.replay();
+
         converter = new AvroConverter();
         converter.setRepository(repository);