Changeset 4264

Timestamp:

2010-09-16 12:48:17 (3 years ago)

Author:

bruno

Message:

• Moved indexerconf from indexer-engine to indexer-model project. This allows indexer-model to perform basic validation of the indexerconf.
• Indexer admin CLI: parse indexerconf against repository to perform full validation. This verifies that the field types and record types specified in the indexerconf exist. The indexer model itself does not do this validation as it does not have/require a connection to a running Lily repository. This validation is optional and can be skipped with the option --force.
• Indexer admin CLI: some general improvements to the output (configure logging, don't print stack traces for expectable exceptions like index conf errors).

Location:

trunk/indexer

Files:

• admin-cli/pom.xml (modified) (1 diff)
• admin-cli/src/main/java/org/lilycms/indexer/admin/cli/AddIndexCli.java (modified) (2 diffs)
• admin-cli/src/main/java/org/lilycms/indexer/admin/cli/BaseIndexerAdminCli.java (modified) (17 diffs)
• admin-cli/src/main/java/org/lilycms/indexer/admin/cli/ListIndexesCli.java (modified) (2 diffs)
• admin-cli/src/main/java/org/lilycms/indexer/admin/cli/TouchIndexCli.java (modified) (2 diffs)
• admin-cli/src/main/java/org/lilycms/indexer/admin/cli/UpdateIndexCli.java (modified) (2 diffs)
• engine/src/main/java/org/lilycms/indexer/conf (deleted)
• engine/src/main/java/org/lilycms/indexer/engine/IndexUpdater.java (moved) (moved from trunk/indexer/engine/src/main/java/org/lilycms/indexer/IndexUpdater.java) (1 diff)
• engine/src/main/java/org/lilycms/indexer/engine/Indexer.java (moved) (moved from trunk/indexer/engine/src/main/java/org/lilycms/indexer/Indexer.java) (2 diffs)
• engine/src/main/java/org/lilycms/indexer/engine/ValueEvaluator.java (modified) (1 diff)
• engine/src/main/java/org/lilycms/indexer/formatter (deleted)
• engine/src/main/resources (deleted)
• engine/src/test/java/org/lilycms/indexer/test/DateYearFormatter.java (modified) (1 diff)
• engine/src/test/java/org/lilycms/indexer/test/IndexerTest.java (modified) (2 diffs)
• engine/src/test/java/org/lilycms/indexer/test/IntegerHierarchyUnderscoreFormatter.java (modified) (1 diff)
• engine/src/test/resources/org/lilycms/indexer/test/indexerconf1.xml (modified) (1 diff)
• fullbuild/src/main/java/org/lilycms/indexer/fullbuild/IndexingMapper.java (modified) (1 diff)
• model/pom.xml (modified) (1 diff)
• model/src/main/java/org/lilycms/indexer/model/impl/IndexerModelImpl.java (modified) (3 diffs)
• model/src/main/java/org/lilycms/indexer/model/indexerconf (added)
• model/src/main/java/org/lilycms/indexer/model/indexerconf/BaseValue.java (added)
• model/src/main/java/org/lilycms/indexer/model/indexerconf/DefaultFormatter.java (added)
• model/src/main/java/org/lilycms/indexer/model/indexerconf/DerefValue.java (added)
• model/src/main/java/org/lilycms/indexer/model/indexerconf/FieldValue.java (added)
• model/src/main/java/org/lilycms/indexer/model/indexerconf/Formatter.java (added)
• model/src/main/java/org/lilycms/indexer/model/indexerconf/Formatters.java (added)
• model/src/main/java/org/lilycms/indexer/model/indexerconf/IndexCase.java (added)
• model/src/main/java/org/lilycms/indexer/model/indexerconf/IndexField.java (added)
• model/src/main/java/org/lilycms/indexer/model/indexerconf/IndexerConf.java (added)
• model/src/main/java/org/lilycms/indexer/model/indexerconf/IndexerConfBuilder.java (added)
• model/src/main/java/org/lilycms/indexer/model/indexerconf/IndexerConfException.java (added)
• model/src/main/java/org/lilycms/indexer/model/indexerconf/Value.java (added)
• model/src/main/resources (added)
• model/src/main/resources/org (added)
• model/src/main/resources/org/lilycms (added)
• model/src/main/resources/org/lilycms/indexer (added)
• model/src/main/resources/org/lilycms/indexer/model (added)
• model/src/main/resources/org/lilycms/indexer/model/indexerconf (added)
• model/src/main/resources/org/lilycms/indexer/model/indexerconf/indexerconf.xsd (copied) (copied from trunk/indexer/engine/src/main/resources/org/lilycms/indexer/conf/indexerconf.xsd)
• worker/src/main/java/org/lilycms/indexer/worker/IndexerWorker.java (modified) (1 diff)

trunk/indexer/admin-cli/pom.xml

@@ -47 +47 @@
       <artifactId>joda-time</artifactId>
     </dependency>
+    <dependency>
+      <groupId>log4j</groupId>
+      <artifactId>log4j</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-log4j12</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.lilycms</groupId>
+      <artifactId>lily-client</artifactId>
+    </dependency>
   </dependencies>
 

trunk/indexer/admin-cli/src/main/java/org/lilycms/indexer/admin/cli/AddIndexCli.java

@@ -38 +38 @@
     }
 
-    public void run(ZooKeeperItf zk, CommandLine cmd) throws Exception {
+    public int run(ZooKeeperItf zk, CommandLine cmd) throws Exception {
         WriteableIndexerModel model = new IndexerModelImpl(zk);
 
@@ -62 +62 @@
 
         System.out.println("Index created: " + indexName);
+
+        return 0;
     }
 

trunk/indexer/admin-cli/src/main/java/org/lilycms/indexer/admin/cli/BaseIndexerAdminCli.java

@@ -3 +3 @@
 import org.apache.commons.cli.*;
 import org.apache.commons.io.FileUtils;
+import org.apache.log4j.ConsoleAppender;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
 import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.Watcher;
+import org.lilycms.client.LilyClient;
 import org.lilycms.indexer.model.api.IndexBatchBuildState;
 import org.lilycms.indexer.model.api.IndexGeneralState;
 import org.lilycms.indexer.model.api.IndexUpdateState;
+import org.lilycms.indexer.model.api.IndexValidityException;
+import org.lilycms.indexer.model.indexerconf.IndexerConfBuilder;
+import org.lilycms.indexer.model.indexerconf.IndexerConfException;
+import org.lilycms.indexer.model.sharding.ShardingConfigException;
 import org.lilycms.util.zookeeper.ZooKeeperImpl;
 import org.lilycms.util.zookeeper.ZooKeeperItf;
 
+import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.net.URI;
@@ -37 +47 @@
 
     public static void start(String[] args, BaseIndexerAdminCli cli) {
+        setupLogging();
+        int result = 1;
         try {
-            cli.runBase(args);
+            System.out.println();
+            result = cli.runBase(args);
+        } catch (IndexValidityException e) {
+            System.out.println("ATTENTION");
+            System.out.println("---------");
+            System.out.println("The index could not be created or updated because:");
+            printExceptionMessages(e);
         } catch (Throwable t) {
             t.printStackTrace();
         }
-    }
-
-    public void runBase(String[] args) throws Exception {
+        System.out.println();
+
+        if (result != 0)
+            System.exit(result);
+    }
+
+    private static void printExceptionMessages(Throwable throwable) {
+        Throwable cause = throwable;
+        while (cause != null) {
+            String prefix = "";
+            if (cause instanceof IndexValidityException) {
+                prefix = "Index definition: ";
+            } else if (cause instanceof IndexerConfException) {
+                prefix = "Indexer configuration: ";
+            } else if (cause instanceof ShardingConfigException) {
+                prefix = "Sharding configuration: ";
+            }
+            System.out.println(prefix + cause.getMessage());
+            cause = cause.getCause();
+        }
+    }
+
+    private static void setupLogging() {
+        Logger rootLogger = Logger.getRootLogger();
+        rootLogger.setLevel(Level.WARN);
+
+        final String CONSOLE_LAYOUT = "[%-5p][%d{ABSOLUTE}][%-10.10t] %c - %m%n";
+
+        ConsoleAppender consoleAppender = new ConsoleAppender();
+        consoleAppender.setLayout(new PatternLayout(CONSOLE_LAYOUT));
+
+        consoleAppender.activateOptions();
+        rootLogger.addAppender(consoleAppender);
+    }
+
+    public int runBase(String[] args) throws Exception {
         Options cliOptions = new Options();
 
@@ -60 +111 @@
                 .create("z");
         cliOptions.addOption(zkOption);
+
+        Option forceOption = OptionBuilder
+                .withDescription("Skips optional validations.")
+                .withLongOpt("force")
+                .create("f");
+        cliOptions.addOption(forceOption);
 
         //
@@ -138 +195 @@
         } catch (ParseException e) {
             System.out.println(e.getMessage());
+            System.out.println();
             showHelp = true;
         }
@@ -146 +204 @@
         if (showHelp || cmd.hasOption(helpOption.getOpt())) {
             printHelp(cliOptions);
-            System.exit(1);
+            return 1;
         }
 
         if (!cmd.hasOption(zkOption.getOpt())) {
-            System.out.println("Zookeeper connection string not specified, using default: " + DEFAULT_ZK_CONNECT);
+            System.out.println("ZooKeeper connection string not specified, using default: " + DEFAULT_ZK_CONNECT);
+            System.out.println();
             zkConnectionString = DEFAULT_ZK_CONNECT;
         } else {
@@ -173 +232 @@
                     System.out.println("SOLR shards should be specified as 'name:URL' pairs, which the following is not:");
                     System.out.println(shardEntry);
-                    System.exit(1);
+                    return 1;
                 }
 
@@ -180 +239 @@
                     System.out.println("Zero-length shard name in the following shard entry:");
                     System.out.println(shardEntry);
-                    System.exit(1);
+                    return 1;
                 }
 
@@ -188 +247 @@
                     if (!uri.isAbsolute()) {
                         System.out.println("Not an absolute URI: " + shardAddress);
-                        System.exit(1);
+                        return 1;
                     }
                 } catch (URISyntaxException e) {
                     System.out.println("Invalid SOLR shard URI: " + shardAddress);
                     System.out.println(e.getMessage());
-                    System.exit(1);
+                    return 1;
                 }
 
                 if (solrShards.containsKey(shardName)) {
                     System.out.println("Duplicate shard name: " + shardName);
-                    System.exit(1);
+                    return 1;
                 }
 
@@ -205 +264 @@
                     System.out.println("Warning: you have two shards pointing to the same URI:");
                     System.out.println(shardAddress);
-                    System.out.println();
                 }
 
@@ -216 +274 @@
                 // Probably cannot occur
                 System.out.println("No SOLR shards specified though option is used.");
-                System.exit(1);
+                return 1;
             }
         }
@@ -226 +284 @@
                 System.out.println("Specified sharding configuration file not found:");
                 System.out.println(configurationFile.getAbsolutePath());
-                System.exit(1);
+                return 1;
             }
 
@@ -238 +296 @@
                 System.out.println("Specified indexer configuration file not found:");
                 System.out.println(configurationFile.getAbsolutePath());
-                System.exit(1);
+                return 1;
             }
 
             indexerConfiguration = FileUtils.readFileToByteArray(configurationFile);
+
+            if (!cmd.hasOption(forceOption.getOpt())) {
+                try {
+                    LilyClient lilyClient = new LilyClient(zkConnectionString);
+                    IndexerConfBuilder.build(new ByteArrayInputStream(indexerConfiguration), lilyClient.getRepository());
+                } catch (Exception e) {
+                    System.out.println(); // separator line because LilyClient might have produced some error logs
+                    System.out.println("Failed to parse & build the indexer configuration.");
+                    System.out.println();
+                    System.out.println("If this problem occurs because no Lily node is available");
+                    System.out.println("or because certain field types or record types do not exist,");
+                    System.out.println("then you can skip this validation using the option --" + forceOption.getLongOpt());
+                    System.out.println();
+                    if (e instanceof IndexerConfException) {
+                        printExceptionMessages(e);
+                    } else {
+                        e.printStackTrace();
+                    }
+                    return 1;
+                }
+            }
         }
 
@@ -250 +329 @@
             } catch (IllegalArgumentException e) {
                 System.out.println("Invalid index state: " + stateName);
-                System.exit(1);
+                return 1;
             }
         }
@@ -260 +339 @@
             } catch (IllegalArgumentException e) {
                 System.out.println("Invalid index update state: " + stateName);
-                System.exit(1);
+                return 1;
             }
         }
@@ -270 +349 @@
             } catch (IllegalArgumentException e) {
                 System.out.println("Invalid index build state: " + stateName);
-                System.exit(1);
+                return 1;
             }
 
             if (buildState != IndexBatchBuildState.BUILD_REQUESTED) {
                 System.out.println("The build state can only be set to " + IndexBatchBuildState.BUILD_REQUESTED);
-                System.exit(1);
+                return 1;
             }
         }
@@ -281 +360 @@
         ZooKeeperItf zk = new ZooKeeperImpl(zkConnectionString, 3000, new MyWatcher());
 
-        run(zk, cmd);
+        return run(zk, cmd);
     }
 
@@ -296 +375 @@
     public abstract List<Option> getOptions();
 
-    public abstract void run(ZooKeeperItf zk, CommandLine cmd) throws Exception;
+    public abstract int run(ZooKeeperItf zk, CommandLine cmd) throws Exception;
 
     private class MyWatcher implements Watcher {

trunk/indexer/admin-cli/src/main/java/org/lilycms/indexer/admin/cli/ListIndexesCli.java

@@ -28 +28 @@
     }
 
-    public void run(ZooKeeperItf zk, CommandLine cmd) throws Exception {
+    public int run(ZooKeeperItf zk, CommandLine cmd) throws Exception {
         WriteableIndexerModel model = new IndexerModelImpl(zk);
 
@@ -63 +63 @@
                 System.out.println("    + Job state: " + lastBatchBuild.getJobState());
             }
+        }
 
-            System.out.println();
-        }
+        return 0;
     }
 }

trunk/indexer/admin-cli/src/main/java/org/lilycms/indexer/admin/cli/TouchIndexCli.java

@@ -31 +31 @@
     }
 
-    public void run(ZooKeeperItf zk, CommandLine cmd) throws Exception {
+    public int run(ZooKeeperItf zk, CommandLine cmd) throws Exception {
         WriteableIndexerModel model = new IndexerModelImpl(zk);
 
         if (!model.hasIndex(indexName)) {
             System.out.println("Index does not exist: " + indexName);
-            System.exit(1);
+            return 1;
         }
 
@@ -48 +48 @@
             model.unlockIndex(lock);
         }
+
+        return 0;
     }
 }

trunk/indexer/admin-cli/src/main/java/org/lilycms/indexer/admin/cli/UpdateIndexCli.java

@@ -39 +39 @@
     }
 
-    public void run(ZooKeeperItf zk, CommandLine cmd) throws Exception {
+    public int run(ZooKeeperItf zk, CommandLine cmd) throws Exception {
         WriteableIndexerModel model = new IndexerModelImpl(zk);
 
         if (!model.hasIndex(indexName)) {
             System.out.println("Index does not exist: " + indexName);
-            System.exit(1);
+            return 1;
         }
 
@@ -94 +94 @@
             model.unlockIndex(lock);
         }
+
+        return 0;
     }
 }

trunk/indexer/engine/src/main/java/org/lilycms/indexer/engine/IndexUpdater.java

@@ -14 +14 @@
  * limitations under the License.
  */
-package org.lilycms.indexer;
+package org.lilycms.indexer.engine;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.metrics.*;
-import org.lilycms.indexer.conf.DerefValue;
-import org.lilycms.indexer.conf.IndexCase;
-import org.lilycms.indexer.conf.IndexField;
+import org.lilycms.indexer.model.indexerconf.DerefValue;
+import org.lilycms.indexer.model.indexerconf.IndexCase;
+import org.lilycms.indexer.model.indexerconf.IndexField;
 import org.lilycms.linkindex.LinkIndex;
 import org.lilycms.repository.api.*;

trunk/indexer/engine/src/main/java/org/lilycms/indexer/engine/Indexer.java

@@ -14 +14 @@
  * limitations under the License.
  */
-package org.lilycms.indexer;
+package org.lilycms.indexer.engine;
 
 import org.apache.commons.logging.Log;
@@ -25 +25 @@
 import org.apache.solr.client.solrj.util.ClientUtils;
 import org.apache.solr.common.SolrInputDocument;
-import org.lilycms.indexer.conf.IndexCase;
-import org.lilycms.indexer.conf.IndexField;
-import org.lilycms.indexer.conf.IndexerConf;
+import org.lilycms.indexer.model.indexerconf.IndexCase;
+import org.lilycms.indexer.model.indexerconf.IndexField;
+import org.lilycms.indexer.model.indexerconf.IndexerConf;
 import org.lilycms.indexer.engine.SolrServers;
 import org.lilycms.indexer.engine.ValueEvaluator;

trunk/indexer/engine/src/main/java/org/lilycms/indexer/engine/ValueEvaluator.java

@@ -8 +8 @@
 import org.apache.tika.parser.Parser;
 import org.apache.tika.sax.BodyContentHandler;
-import org.lilycms.indexer.conf.DerefValue;
-import org.lilycms.indexer.conf.DerefValue.Follow;
-import org.lilycms.indexer.conf.DerefValue.FieldFollow;
-import org.lilycms.indexer.conf.DerefValue.VariantFollow;
-import org.lilycms.indexer.conf.DerefValue.MasterFollow;
-import org.lilycms.indexer.conf.FieldValue;
-import org.lilycms.indexer.conf.IndexerConf;
-import org.lilycms.indexer.conf.Value;
-import org.lilycms.indexer.formatter.Formatter;
+import org.lilycms.indexer.model.indexerconf.DerefValue;
+import org.lilycms.indexer.model.indexerconf.DerefValue.Follow;
+import org.lilycms.indexer.model.indexerconf.DerefValue.FieldFollow;
+import org.lilycms.indexer.model.indexerconf.DerefValue.VariantFollow;
+import org.lilycms.indexer.model.indexerconf.DerefValue.MasterFollow;
+import org.lilycms.indexer.model.indexerconf.FieldValue;
+import org.lilycms.indexer.model.indexerconf.IndexerConf;
+import org.lilycms.indexer.model.indexerconf.Value;
+import org.lilycms.indexer.model.indexerconf.Formatter;
 import org.lilycms.repository.api.*;
 import org.lilycms.util.io.Closer;

trunk/indexer/engine/src/test/java/org/lilycms/indexer/test/DateYearFormatter.java

@@ -18 +18 @@
 import org.joda.time.DateTime;
 import org.joda.time.LocalDate;
-import org.lilycms.indexer.formatter.DefaultFormatter;
+import org.lilycms.indexer.model.indexerconf.DefaultFormatter;
 import org.lilycms.repository.api.ValueType;
 

trunk/indexer/engine/src/test/java/org/lilycms/indexer/test/IndexerTest.java

@@ -18 +18 @@
 import static org.junit.Assert.assertEquals;
 
-import org.lilycms.indexer.IndexUpdater;
+import org.lilycms.indexer.engine.IndexUpdater;
+import org.lilycms.indexer.engine.Indexer;
 import org.lilycms.indexer.engine.SolrServers;
+import org.lilycms.indexer.model.indexerconf.IndexerConfBuilder;
 import org.lilycms.linkindex.LinkIndexUpdater;
 import org.lilycms.util.repo.RecordEvent;
@@ -40 +42 @@
 import org.junit.Test;
 import org.lilycms.hbaseindex.IndexManager;
-import org.lilycms.indexer.Indexer;
-import org.lilycms.indexer.conf.IndexerConf;
-import org.lilycms.indexer.conf.IndexerConfBuilder;
+import org.lilycms.indexer.model.indexerconf.IndexerConf;
 import org.lilycms.linkindex.LinkIndex;
 import org.lilycms.repository.api.*;

trunk/indexer/engine/src/test/java/org/lilycms/indexer/test/IntegerHierarchyUnderscoreFormatter.java

@@ -16 +16 @@
 package org.lilycms.indexer.test;
 
-import org.lilycms.indexer.formatter.Formatter;
+import org.lilycms.indexer.model.indexerconf.Formatter;
 import org.lilycms.repository.api.HierarchyPath;
 import org.lilycms.repository.api.ValueType;

trunk/indexer/engine/src/test/resources/org/lilycms/indexer/test/indexerconf1.xml

@@ -11 +11 @@
 
     <!-- Default formatter: handles any kind of value. -->
-    <formatter class="org.lilycms.indexer.formatter.DefaultFormatter"/>
+    <formatter class="org.lilycms.indexer.model.indexerconf.DefaultFormatter"/>
 
     <!-- The formatters below will only be used when referred to explictly by name, because the

trunk/indexer/fullbuild/src/main/java/org/lilycms/indexer/fullbuild/IndexingMapper.java

@@ -9 +9 @@
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.mapreduce.TableMapper;
-import org.lilycms.indexer.Indexer;
-import org.lilycms.indexer.conf.IndexerConf;
-import org.lilycms.indexer.conf.IndexerConfBuilder;
+import org.lilycms.indexer.engine.Indexer;
+import org.lilycms.indexer.model.indexerconf.IndexerConf;
+import org.lilycms.indexer.model.indexerconf.IndexerConfBuilder;
 import org.lilycms.indexer.engine.SolrServers;
 import org.lilycms.indexer.model.sharding.DefaultShardSelectorBuilder;

trunk/indexer/model/pom.xml

@@ -43 +43 @@
     <dependency>
       <groupId>org.lilycms</groupId>
+      <artifactId>lily-repo-util</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.lilycms</groupId>
       <artifactId>lily-repository-api</artifactId>
     </dependency>

trunk/indexer/model/src/main/java/org/lilycms/indexer/model/impl/IndexerModelImpl.java

@@ -7 +7 @@
 import static org.apache.zookeeper.Watcher.Event.EventType.*;
 import org.lilycms.indexer.model.api.*;
+import org.lilycms.indexer.model.indexerconf.IndexerConfBuilder;
+import org.lilycms.indexer.model.indexerconf.IndexerConfException;
 import org.lilycms.indexer.model.sharding.JsonShardSelectorBuilder;
 import org.lilycms.indexer.model.sharding.ShardSelector;
@@ -14 +16 @@
 import static org.lilycms.indexer.model.api.IndexerModelEventType.*;
 
+import java.io.ByteArrayInputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -161 +164 @@
             }
         }
+
+        try {
+            IndexerConfBuilder.validate(new ByteArrayInputStream(index.getConfiguration()));
+        } catch (IndexerConfException e) {
+            throw new IndexValidityException("The indexer configuration is not XML well-formed or valid.", e);
+        }
     }
 

trunk/indexer/worker/src/main/java/org/lilycms/indexer/worker/IndexerWorker.java

@@ -5 +5 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.lilycms.indexer.IndexUpdater;
-import org.lilycms.indexer.Indexer;
-import org.lilycms.indexer.conf.IndexerConf;
-import org.lilycms.indexer.conf.IndexerConfBuilder;
+import org.lilycms.indexer.engine.IndexUpdater;
+import org.lilycms.indexer.engine.Indexer;
+import org.lilycms.indexer.model.indexerconf.IndexerConf;
+import org.lilycms.indexer.model.indexerconf.IndexerConfBuilder;
 import org.lilycms.indexer.engine.SolrServers;
 import org.lilycms.indexer.model.sharding.DefaultShardSelectorBuilder;