Changeset 4623

Timestamp:

2011-02-03 13:55:57 (2 years ago)

Author:

bruno

Message:

Make it possible to configure the initial setting for maxFileSize and memStoreFlushSize per created HBase table (via tables.xml).
Upgrade to newer Kauri snapshot to make use of improved Conf API (allow null as default value).

Location:

trunk

Files:

• cr/process/server/conf/general/tables.xml (modified) (1 diff)
• cr/process/server/general-module/src/main/java/org/lilyproject/server/modules/general/TableConfigBuilder.java (modified) (1 diff)
• global/hbase-util/src/main/java/org/lilyproject/util/hbase/HBaseTableFactory.java (modified) (1 diff)
• global/hbase-util/src/main/java/org/lilyproject/util/hbase/HBaseTableFactoryImpl.java (modified) (2 diffs)
• global/hbase-util/src/main/java/org/lilyproject/util/hbase/TableConfig.java (modified) (2 diffs)
• global/hbaseindex/src/main/java/org/lilyproject/hbaseindex/IndexManager.java (modified) (1 diff)
• pom.xml (modified) (1 diff)

trunk/cr/process/server/conf/general/tables.xml

@@ -23 +23 @@
       <regionCount>20</regionCount>
       <splitKeys></splitKeys>
+
+      <!-- Maximum file size in bytes -->
+      <!--
+      <maxFileSize>67108864</maxFileSize>
+      -->
+
+      <!-- Memstore flush size in bytes -->
+      <!--
+      <memStoreFlushSize>33554432</memStoreFlushSize>
+      -->
     </splits>
   </table>

trunk/cr/process/server/general-module/src/main/java/org/lilyproject/server/modules/general/TableConfigBuilder.java

@@ -21 +21 @@
             byte[] splitKeyPrefixBytes = splitKeyPrefix != null ? Bytes.toBytesBinary(splitKeyPrefix) : null;
 
+            Long maxFileSize = table.getChild("maxFileSize").getValueAsLong(null);
+            Long memStoreFlushSize = table.getChild("memStoreFlushSize").getValueAsLong(null);
+
             TableConfig config = new TableConfig(regionCount, splitKeys, splitKeyPrefixBytes);
+            config.setMaxFileSize(maxFileSize);
+            config.setMemStoreFlushSize(memStoreFlushSize);
 
             result.put(new ByteArrayKey(tableName), config);

trunk/global/hbase-util/src/main/java/org/lilyproject/util/hbase/HBaseTableFactory.java

@@ -24 +24 @@
     TableConfig getTableConfig(byte[] tableName);
     
+    void configure(HTableDescriptor tableDescriptor);
+
     /**
      *

trunk/global/hbase-util/src/main/java/org/lilyproject/util/hbase/HBaseTableFactoryImpl.java

@@ -63 +63 @@
 
             try {
+                // Make a deep copy, we don't want to touch the original
+                tableDescriptor = new HTableDescriptor(tableDescriptor);
+                configure(tableDescriptor);
+
                 int regionCount = splitKeys == null ? 1 : splitKeys.length + 1;
                 log.info("Creating '" + tableDescriptor.getNameAsString() + "' table using "
-                        + regionCount + " initial region" + (regionCount > 1 ? "s.": "."));
+                        + regionCount + " initial region" + (regionCount > 1 ? "s." : "."));
                 admin.createTable(tableDescriptor, splitKeys);
             } catch (TableExistsException e2) {
@@ -76 +80 @@
 
         return new LocalHTable(configuration, tableDescriptor.getName());
+    }
+
+    public void configure(HTableDescriptor tableDescriptor) {
+        Long maxFileSize = getTableConfig(tableDescriptor.getName()).getMaxFileSize();
+        if (maxFileSize != null)
+            tableDescriptor.setMaxFileSize(maxFileSize);
+
+        Long memStoreFlushSize = getTableConfig(tableDescriptor.getName()).getMemStoreFlushSize();
+        if (memStoreFlushSize != null)
+            tableDescriptor.setMemStoreFlushSize(memStoreFlushSize);
     }
     

trunk/global/hbase-util/src/main/java/org/lilyproject/util/hbase/TableConfig.java

@@ -9 +9 @@
     private String splitKeysAsString;
     private byte[] splitKeyPrefix;
+    private Long maxFileSize;
+    private Long memStoreFlushSize;
 
     public TableConfig() {
@@ -24 +26 @@
         this.splitKeysAsString = splitKeys;
         this.splitKeyPrefix = splitKeyPrefix == null ? new byte[0] : splitKeyPrefix;
+    }
+
+    public Long getMaxFileSize() {
+        return maxFileSize;
+    }
+
+    public void setMaxFileSize(Long maxFileSize) {
+        this.maxFileSize = maxFileSize;
+    }
+
+    public Long getMemStoreFlushSize() {
+        return memStoreFlushSize;
+    }
+
+    public void setMemStoreFlushSize(Long memStoreFlushSize) {
+        this.memStoreFlushSize = memStoreFlushSize;
     }
 

trunk/global/hbaseindex/src/main/java/org/lilyproject/hbaseindex/IndexManager.java

@@ -103 +103 @@
         table.addFamily(family);
 
+        tableFactory.configure(table);
+
         if (splitKeys == null) {
             splitKeys = tableFactory.getSplitKeys(table.getName());

trunk/pom.xml

@@ -33 +33 @@
     <!-- The following properties are all Kauri-related, and should be brought in sync with
          Kauri when moving to a new Kauri version. -->
-    <version.kauri>0.4-r1771</version.kauri>
+    <version.kauri>0.4-r1835</version.kauri>
     <version.spring>2.5.6</version.spring>
     <version.jaxrs>1.0</version.jaxrs>