Changeset 4130

Timestamp:

2010-07-09 12:18:02 (3 years ago)

Author:

bruno

Message:

Import tool:

• add support for non-string value types (all except blob)
• support user-defined IDs for records & record updating
• allow unquoted field names and comments in the "json"

Updated to newer HBase snapshot for no particular reason.

Updated to latest SOLR, just because we can.

FieldTypeNotFoundException?: fix 'todo' in case of getFieldTypeByName

Location:

projects/lily/trunk

Files:

• README.txt (modified) (2 diffs)
• indexer/pom.xml (modified) (2 diffs)
• pom.xml (modified) (1 diff)
• process/server/conf/hbase/hbase.xml (modified) (1 diff)
• process/server/conf/repository/repository.xml (modified) (1 diff)
• repo-util/src/main/java/org/lilycms/repoutil/JsonUtil.java (modified) (1 diff)
• repository/api/src/main/java/org/lilycms/repository/api/FieldTypeNotFoundException.java (modified) (2 diffs)
• repository/api/src/main/java/org/lilycms/repository/api/Record.java (modified) (1 diff)
• repository/impl/src/main/avro/lily.avpr (modified) (1 diff)
• repository/impl/src/main/java/org/lilycms/repository/avro/AvroConverter.java (modified) (3 diffs)
• repository/impl/src/main/java/org/lilycms/repository/impl/HBaseTypeManager.java (modified) (1 diff)
• repository/impl/src/main/java/org/lilycms/repository/impl/IdRecordImpl.java (modified) (1 diff)
• repository/impl/src/main/java/org/lilycms/repository/impl/RecordImpl.java (modified) (2 diffs)
• tools/import/sample.json (modified) (1 diff)
• tools/import/src/main/java/org/lilycms/tools/import_/DefaultImportListener.java (modified) (2 diffs)
• tools/import/src/main/java/org/lilycms/tools/import_/EntityType.java (modified) (1 diff)
• tools/import/src/main/java/org/lilycms/tools/import_/ImportListener.java (modified) (1 diff)
• tools/import/src/main/java/org/lilycms/tools/import_/ImportTool.java (modified) (2 diffs)
• tools/import/src/main/java/org/lilycms/tools/import_/JsonImportTool.java (modified) (4 diffs)

projects/lily/trunk/README.txt

@@ -14 +14 @@
 -------------
 
-Download Solr 1.4.0 from http://lucene.apache.org/solr/ and extract it somewhere.
+Download Solr 1.4.1 from http://lucene.apache.org/solr/ and extract it somewhere.
 
 Configuring settings.xml
@@ -30 +30 @@
 
       <properties>
-        <solr.war>/home/bruno/projects/lily-trunk/deps/apache-solr-1.4.0/dist/apache-solr-1.4.0.war</solr.war>
+        <solr.war>/path/to/apache-solr-1.4.1/dist/apache-solr-1.4.1.war</solr.war>
       </properties>
     </profile>

projects/lily/trunk/indexer/pom.xml

@@ -77 +77 @@
       <groupId>org.apache.solr</groupId>
       <artifactId>solr-solrj</artifactId>
-      <version>1.4.0</version>
+      <version>1.4.1</version>
     </dependency>
 
@@ -83 +83 @@
       <groupId>org.apache.solr</groupId>
       <artifactId>solr-core</artifactId>
-      <version>1.4.0</version>
+      <version>1.4.1</version>
       <scope>test</scope>
       <exclusions>

projects/lily/trunk/pom.xml

@@ -23 +23 @@
 
   <properties>
-    <hbase.version>0.21.0-r959305</hbase.version>
+    <hbase.version>0.21.0-r962435</hbase.version>
     <!-- When changing the hbase version, also change the hadoop version below to match
          what is in HBase's pom.xml -->

projects/lily/trunk/process/server/conf/hbase/hbase.xml

@@ -2 +2 @@
 <hbase xmlns:conf="http://kauriproject.org/configuration"
        conf:inherit="shallow">
-  <!--
   <zookeeperQuorum>localhost</zookeeperQuorum>
   <zookeeperClientPort>2181</zookeeperClientPort>
-  -->
 </hbase>

projects/lily/trunk/process/server/conf/repository/repository.xml

@@ -2 +2 @@
 <repository xmlns:conf="http://kauriproject.org/configuration"
             conf:inherit="shallow">
-  <!--
   <zookeeperConnectString>localhost:2181</zookeeperConnectString>
-  -->
 
   <!-- Filesystem to use to store blobs -->

projects/lily/trunk/repo-util/src/main/java/org/lilycms/repoutil/JsonUtil.java

@@ -41 +41 @@
     }
 
+    public static boolean getBoolean(JsonNode node, String prop) throws JsonFormatException {
+        if (node.get(prop) == null) {
+            throw new JsonFormatException("Missing required property: " + prop);
+        }
+        if (!node.get(prop).isBoolean()) {
+            throw new JsonFormatException("Not a string property: " + prop);
+        }
+        return node.get(prop).getBooleanValue();
+    }
+
     public static int getInt(JsonNode node, String prop, int defaultValue) throws JsonFormatException {
         if (node.get(prop) == null) {

projects/lily/trunk/repository/api/src/main/java/org/lilycms/repository/api/FieldTypeNotFoundException.java

@@ -22 +22 @@
 
     private final String id;
+    private final QName name;
     private final Long version;
 
     public FieldTypeNotFoundException(String id, Long version) {
         this.id = id;
+        this.name = null;
         this.version = version;
     }
     
+    public FieldTypeNotFoundException(QName name, Long version) {
+        this.id = null;
+        this.name = name;
+        this.version = version;
+    }
+
     public String getId() {
         return id;
     }
-    
+
+    public QName getName() {
+        return name;
+    }
+
     public Long getVersion() {
         return version;
@@ -41 +53 @@
         StringBuilder stringBuilder = new StringBuilder();
         stringBuilder.append("FieldType <");
-        stringBuilder.append(id);
+        stringBuilder.append(id != null ? id : name);
         stringBuilder.append("> ");
         if (version != null) {

projects/lily/trunk/repository/api/src/main/java/org/lilycms/repository/api/Record.java

@@ -161 +161 @@
     
     boolean equals(Object obj);
+
+    /**
+     * Compares the two records, ignoring some aspects. This method is intended to compare
+     * a record loaded from the repository with a newly instantiated one in which some things
+     * are typically not supplied.
+     *
+     * <p>The aspects which are ignored are:
+     * <ul>
+     *   <li>the version
+     *   <li>the record types: only the name of the non-versioned record type is compared, if
+     *       it is not null in both. The version of the record type is not compared.
+     * </ul>
+     */
+    boolean softEquals(Object obj);
 }

projects/lily/trunk/repository/impl/src/main/avro/lily.avpr

@@ -179 +179 @@
       "type": "error",
       "fields": [
-        {"name": "id", "type": "string"},
+        {"name": "id", "type": ["string", "null"]},
+        {"name": "name", "type": ["AvroQName", "null"]},
         {"name": "version", "type": "long"},
         {"name": "remoteCauses", "type": ["null", {"type": "array", "items": "AvroExceptionCause"}]}

projects/lily/trunk/repository/impl/src/main/java/org/lilycms/repository/avro/AvroConverter.java

@@ -248 +248 @@
 
     public AvroQName convert(QName name) {
+        if (name == null)
+            return null;
+
         AvroQName avroQName = new AvroQName();
         if (name.getNamespace() != null) {
@@ -328 +331 @@
     public AvroFieldTypeNotFoundException convert(FieldTypeNotFoundException exception) {
         AvroFieldTypeNotFoundException avroException = new AvroFieldTypeNotFoundException();
-        avroException.id = new Utf8(exception.getId());
+        if (exception.getId() != null)
+            avroException.id = new Utf8(exception.getId());
+        if (exception.getName() != null)
+            avroException.name = convert(exception.getName());
         Long version = exception.getVersion();
         if (version != null) {
@@ -362 +368 @@
 
     public FieldTypeNotFoundException convert(AvroFieldTypeNotFoundException avroException) {
-        FieldTypeNotFoundException exception = new FieldTypeNotFoundException(convert(avroException.id), avroException.version);
+        FieldTypeNotFoundException exception;
+        if (avroException.id != null) {
+            exception = new FieldTypeNotFoundException(convert(avroException.id), avroException.version);
+        } else {
+            exception = new FieldTypeNotFoundException(convert(avroException.name), avroException.version);
+        }
         restoreCauses(avroException.remoteCauses, exception);
         return exception;

projects/lily/trunk/repository/impl/src/main/java/org/lilycms/repository/impl/HBaseTypeManager.java

@@ -495 +495 @@
         // TODO the below is a temporary fix, should probably be fixed in getFieldTypeFromCache
         if (fieldType == null) {
-            throw new FieldTypeNotFoundException("todo", 1L);
+            throw new FieldTypeNotFoundException(name, 1L);
         }
         return fieldType;

projects/lily/trunk/repository/impl/src/main/java/org/lilycms/repository/impl/IdRecordImpl.java

@@ -137 +137 @@
         throw new UnsupportedOperationException("IdRecordImpl does not support equals.");
     }
+
+    public boolean softEquals(Object obj) {
+        throw new UnsupportedOperationException("IdRecordImpl does not support softEquals.");
+    }
 }

projects/lily/trunk/repository/impl/src/main/java/org/lilycms/repository/impl/RecordImpl.java

@@ -164 +164 @@
     @Override
     public boolean equals(Object obj) {
+        if (!softEquals(obj))
+            return false;
+
+        RecordImpl other = (RecordImpl) obj;
+
+        if (recordTypeIds == null) {
+            if (other.recordTypeIds != null)
+                return false;
+        } else if (!recordTypeIds.equals(other.recordTypeIds)) {
+            return false;
+        }
+
+        if (recordTypeVersions == null) {
+            if (other.recordTypeVersions != null)
+                return false;
+        } else if (!recordTypeVersions.equals(other.recordTypeVersions)) {
+            return false;
+        }
+
+        if (version == null) {
+            if (other.version != null)
+                return false;
+        } else if (!version.equals(other.version)) {
+            return false;
+        }
+
+        return true;
+    }
+
+    public boolean softEquals(Object obj) {
         if (this == obj)
             return true;
@@ -171 +201 @@
             return false;
         RecordImpl other = (RecordImpl) obj;
+
         if (fields == null) {
             if (other.fields != null)
                 return false;
-        } else if (!fields.equals(other.fields))
-            return false;
+        } else if (!fields.equals(other.fields)) {
+            return false;
+        }
+
         if (fieldsToDelete == null) {
             if (other.fieldsToDelete != null)
                 return false;
-        } else if (!fieldsToDelete.equals(other.fieldsToDelete))
-            return false;
+        } else if (!fieldsToDelete.equals(other.fieldsToDelete)) {
+            return false;
+        }
+
         if (id == null) {
             if (other.id != null)
                 return false;
-        } else if (!id.equals(other.id))
-            return false;
-        if (recordTypeIds == null) {
-            if (other.recordTypeIds != null)
-                return false;
-        } else if (!recordTypeIds.equals(other.recordTypeIds))
-            return false;
-        if (recordTypeVersions == null) {
-            if (other.recordTypeVersions != null)
-                return false;
-        } else if (!recordTypeVersions.equals(other.recordTypeVersions))
-            return false;
-        if (version == null) {
-            if (other.version != null)
-                return false;
-        } else if (!version.equals(other.version))
-            return false;
+        } else if (!id.equals(other.id)) {
+            return false;
+        }
+
+        String nonVersionedRT1 = recordTypeIds.get(Scope.NON_VERSIONED);
+        String nonVersionedRT2 = other.recordTypeIds.get(Scope.NON_VERSIONED);
+
+        if (nonVersionedRT1 != null && nonVersionedRT2 != null && !nonVersionedRT1.equals(nonVersionedRT2)) {
+            return false;
+        }
+
         return true;
     }

projects/lily/trunk/tools/import/sample.json

@@ +1 @@
+/*
+  This is a sample input file for the import tool.
+
+  The syntax is not limited to strict json: unquoted property names
+  and comments are allowed.
+*/
 {
-  "namespaces": [
-    { "prefix": "sample", "uri": "org.lilycms.tools.import_.sample"}
+  namespaces: [
+    { prefix: "sample", uri: "org.lilycms.tools.import_.sample"}
   ],
-  "fieldTypes": [
+  fieldTypes: [
     {
-      "name": "sample:field1",
-      "valueType": { "primitive": "STRING" },
-      "scope": "versioned"
+      name: "sample:stringfield",
+      valueType: { "primitive": "STRING" },
+      scope: "versioned"
     },
     {
-      "name": "sample:field2",
-      "valueType": { "primitive": "STRING" },
-      "scope": "non_versioned"
+      name: "sample:multiValueString",
+      valueType: { "primitive": "STRING", multiValue: true },
+      scope: "non_versioned"
+    },
+    {
+      name: "sample:intfield",
+      valueType: { "primitive": "INTEGER" },
+      scope: "non_versioned"
+    },
+    {
+      name: "sample:longfield",
+      valueType: { "primitive": "LONG" },
+      scope: "non_versioned"
+    },
+    {
+      name: "sample:booleanfield",
+      valueType: { "primitive": "BOOLEAN" },
+      scope: "non_versioned"
+    },
+    {
+      name: "sample:linkfield",
+      valueType: { "primitive": "LINK" },
+      scope: "non_versioned"
+    },
+    {
+      name: "sample:datefield",
+      valueType: { "primitive": "DATE" },
+      scope: "non_versioned"
+    },
+    {
+      name: "sample:datetimefield",
+      valueType: { "primitive": "DATETIME" },
+      scope: "non_versioned"
     }
   ],
-  "recordTypes": [
+  recordTypes: [
     {
-      "name": "Book",
-      "fields": [
-        {"name": "sample:field1", "mandatory": false },
-        {"name": "sample:field2", "mandatory": false }
+      name: "MiscFields",
+      fields: [
+        {name: "sample:field1", mandatory: false },
+        {name: "sample:field2", mandatory: false }
       ]
     }
   ],
-  "records": [
-    {"type": "Book", "sample:field1": "My first document."},
-    {"type": "Book", "sample:field1": "My second document.", "sample:field2": "some more data"}
+  records: [
+    {
+      type: "MiscFields",
+      "sample:stringfield": "My first document.",
+      "sample:multiValueString": ["first", "second", "third"],
+      "sample:intfield": 23,
+      "sample:longfield": 242,
+      "sample:datefield": "2010-07-09",
+      "sample:datetimefield": "2010-07-09T14:04:46",
+      "sample:booleanfield": true
+    },
+    {
+      type: "MiscFields",
+      id: "zoo",
+      "sample:stringfield": "A document with a user-defined ID"
+    },
+    {
+      type: "MiscFields",
+      "sample:stringfield": "A document with a link field",
+      "sample:linkfield": "USER.zoo"
+    }
   ]
 }

projects/lily/trunk/tools/import/src/main/java/org/lilycms/tools/import_/DefaultImportListener.java

@@ -20 +20 @@
     }
 
-    public void existsAndEqual(EntityType entityType, String entityName) {
-        out.println(String.format("%1$s already exists and is equal: %2$s", toText(entityType), entityName));
+    public void existsAndEqual(EntityType entityType, String entityName, String entityId) {
+        out.println(String.format("%1$s already exists and is equal: %2$s", toText(entityType), id(entityName, entityId)));
     }
 
-    public void updated(EntityType entityType, String entityName, String entityId) {
-        out.println(String.format("%1$s updated: %2$s", toText(entityType), entityName));
+    public void updated(EntityType entityType, String entityName, String entityId, long version) {
+        if (entityType == EntityType.RECORD || entityType == EntityType.RECORD_TYPE) {
+            out.println(String.format("%1$s updated: %2$s (version %3$s)", toText(entityType), id(entityName, entityId), version));
+        } else {
+            out.println(String.format("%1$s updated: %2$s", toText(entityType), id(entityName, entityId)));
+        }
     }
 
     public void created(EntityType entityType, String entityName, String entityId) {
-        out.println(String.format("%1$s created: %2$s", toText(entityType), entityName));                        
+        out.println(String.format("%1$s created: %2$s", toText(entityType), id(entityName, entityId)));
+    }
+
+    private String id(String entityName, String entityId) {
+        if (entityName != null) {
+            return entityName;
+        } else {
+            return entityId;
+        }
     }
 
@@ -41 +53 @@
                 entityTypeName = "Record type";
                 break;
+            case RECORD:
+                entityTypeName = "Record";
+                break;
             default:
                 throw new RuntimeException("Unexpected entity type: " + entityType);

projects/lily/trunk/tools/import/src/main/java/org/lilycms/tools/import_/EntityType.java

@@ -2 +2 @@
 
 public enum EntityType {
-    FIELD_TYPE, RECORD_TYPE
+    FIELD_TYPE, RECORD_TYPE, RECORD
 }

projects/lily/trunk/tools/import/src/main/java/org/lilycms/tools/import_/ImportListener.java

@@ -5 +5 @@
             throws ImportConflictException;
 
-    void existsAndEqual(EntityType entityType, String entityName);
+    void existsAndEqual(EntityType entityType, String entityName, String entityId);
 
-    void updated(EntityType entityType, String entityName, String entityId);
+    void updated(EntityType entityType, String entityName, String entityId, long version);
 
     void created(EntityType entityType, String entityName, String entityId);    

projects/lily/trunk/tools/import/src/main/java/org/lilycms/tools/import_/ImportTool.java

@@ -43 +43 @@
 
             // everything equal, skip it
-            importListener.existsAndEqual(EntityType.FIELD_TYPE, newFieldType.getName().toString());
+            importListener.existsAndEqual(EntityType.FIELD_TYPE, newFieldType.getName().toString(), null);
             return oldFieldType;
         }
@@ -85 +85 @@
             if (updated) {
                 oldRecordType = typeManager.updateRecordType(oldRecordType);
-                importListener.updated(EntityType.RECORD_TYPE, oldRecordType.getId(), null);
+                importListener.updated(EntityType.RECORD_TYPE, null, oldRecordType.getId(), oldRecordType.getVersion());
             } else {
-                importListener.existsAndEqual(EntityType.RECORD_TYPE, oldRecordType.getId());
+                importListener.existsAndEqual(EntityType.RECORD_TYPE, null, oldRecordType.getId());
             }
             return oldRecordType;
         } else {
             RecordType createdRecordType = typeManager.createRecordType(newRecordType);
-            importListener.created(EntityType.RECORD_TYPE, createdRecordType.getId(), null);
+            importListener.created(EntityType.RECORD_TYPE, null, createdRecordType.getId());
             return createdRecordType;
+        }
+    }
+
+    public Record importRecord(Record newRecord) throws RepositoryException {
+        Record oldRecord = null;
+        if (newRecord.getId() != null) {
+            try {
+                oldRecord = repository.read(newRecord.getId());
+            } catch (RecordNotFoundException e) {
+                // ok
+            }
+        }
+
+        if (oldRecord != null) {
+            if (newRecord.softEquals(oldRecord)) {
+                importListener.existsAndEqual(EntityType.RECORD, null, newRecord.getId().toString());
+                return oldRecord;
+            } else {
+                // Delete fields which are not present in the new record anymore
+                for (Map.Entry<QName, Object> field : oldRecord.getFields().entrySet()) {
+                    if (!newRecord.hasField(field.getKey())) {
+                        newRecord.delete(field.getKey(), true);
+                    }
+                }
+                Record updatedRecord = repository.update(newRecord);
+                importListener.updated(EntityType.RECORD, null, updatedRecord.getId().toString(), updatedRecord.getVersion());
+                return updatedRecord;
+            }
+        } else {
+            Record createdRecord = repository.create(newRecord);
+            importListener.created(EntityType.RECORD, null, createdRecord.getId().toString());
+            return createdRecord;
         }
     }

projects/lily/trunk/tools/import/src/main/java/org/lilycms/tools/import_/JsonImportTool.java

@@ -6 +6 @@
 import org.codehaus.jackson.JsonToken;
 import org.codehaus.jackson.map.MappingJsonFactory;
+import org.joda.time.DateTime;
+import org.joda.time.LocalDate;
 import org.lilycms.client.Client;
 import org.lilycms.repository.api.*;
@@ -61 +63 @@
         namespaces.clear();
 
-        JsonFactory f = new MappingJsonFactory();
-        JsonParser jp = f.createJsonParser(is);
+        JsonFactory jsonFactory = new MappingJsonFactory();
+        jsonFactory.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
+        jsonFactory.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
+        JsonParser jp = jsonFactory.createJsonParser(is);
 
         JsonToken current;
@@ -166 +170 @@
         Record record = repository.newRecord();
 
+        String id = getString(node, "id", null);
+        if (id != null) {
+            record.setId(repository.getIdGenerator().newRecordId(id));
+        }
+
         String type = getString(node, "type");
         record.setRecordType(type);
@@ -174 +183 @@
             if (name.contains(":")) {
                 QName qname = parseQName(name);
-                String value = getString(node, name);
+                ValueType valueType = typeManager.getFieldTypeByName(qname).getValueType();
+                Object value = readMultiValue(getNode(node, name), valueType, name);
                 record.setField(qname, value);
             }
         }
 
-        record = repository.create(record);
-        System.out.println("Created record " + record.getId());
+        importTool.importRecord(record);
+    }
+
+    private Object readMultiValue(JsonNode node, ValueType valueType, String prop) throws ImportException {
+        if (valueType.isMultiValue()) {
+            if (!node.isArray()) {
+                throw new ImportException("Multi-value value should be specified as array in " + prop);
+            }
+
+            List<Object> value = new ArrayList<Object>();
+            for (int i = 0; i < node.size(); i++) {
+                value.add(readHierarchical(node.get(i), valueType, prop));
+            }
+
+            return value;
+        } else {
+            return readHierarchical(node, valueType, prop);
+        }
+    }
+
+    private Object readHierarchical(JsonNode node, ValueType valueType, String prop) throws ImportException {
+        if (valueType.isHierarchical()) {
+            if (!node.isArray()) {
+                throw new ImportException("Hierarchical value should be specified as an array in " + prop);
+            }
+
+            Object[] elements = new Object[node.size()];
+            for (int i = 0; i < node.size(); i++) {
+                elements[i] = readPrimitive(node.get(i), valueType, prop);
+            }
+
+            return new HierarchyPath(elements);
+        } else {
+            return readPrimitive(node, valueType, prop);
+        }
+    }
+
+    private Object readPrimitive(JsonNode node, ValueType valueType, String prop) throws ImportException {
+        String primitive = valueType.getPrimitive().getName();
+
+        if (primitive.equals("STRING")) {
+            if (!node.isTextual())
+                throw new ImportException("Expected text value for " + prop);
+
+            return node.getTextValue();
+        } else if (primitive.equals("INTEGER")) {
+            if (!node.isIntegralNumber())
+                throw new ImportException("Expected int value for " + prop);
+
+            return node.getIntValue();
+        } else if (primitive.equals("LONG")) {
+            if (!node.isIntegralNumber())
+                throw new ImportException("Expected long value for " + prop);
+
+            return node.getLongValue();
+        } else if (primitive.equals("BOOLEAN")) {
+            if (!node.isBoolean())
+                throw new ImportException("Expected boolean value for " + prop);
+
+            return node.getBooleanValue();
+        } else if (primitive.equals("LINK")) {
+            if (!node.isTextual())
+                throw new ImportException("Expected text value for " + prop);
+            
+            return Link.fromString(node.getTextValue(), repository.getIdGenerator());
+        } else if (primitive.equals("DATE")) {
+            if (!node.isTextual())
+                throw new ImportException("Expected text value for " + prop);
+
+            return new LocalDate(node.getTextValue());
+        } else if (primitive.equals("DATETIME")) {
+            if (!node.isTextual())
+                throw new ImportException("Expected text value for " + prop);
+
+            return new DateTime(node.getTextValue());
+        } else {
+            throw new ImportException("Primitive value type not supported by import tool: " + primitive);
+        }
     }