Changeset 5705

Timestamp:

2010-08-25 14:01:51 (3 years ago)

Author:

karel

Message:

Lucene analyzer plug & play...

• Allow plugging in multiple lucene analyzers. (via the pluginRegistry)
• Allow specifying the analyzers to be used for indexing via myconfig.xml
• Allow specifying the default query analyzer via myconfig.xml
• Allow specifying an alternative query analyzer via query option ft_query_analyzer

Location:

trunk/daisy/repository/server/src/main/java/org/outerj/daisy

Files:

• ftindex/FullTextIndex.java (modified) (1 diff)
• ftindex/FullTextIndexImpl.java (modified) (11 diffs)
• query/model/Query.java (modified) (3 diffs)
• repository/serverimpl/query/LocalQueryManager.java (modified) (1 diff)

trunk/daisy/repository/server/src/main/java/org/outerj/daisy/ftindex/FullTextIndex.java

@@ -27 +27 @@
      * This should be guaranteed to happen, i.o.w. use a try-finally block.</b></p>
      */
-    Hits search(String query, long branchId, long languageId, Date date, boolean searchName, boolean searchContent, boolean searchFields) throws QueryException;
+    Hits search(String analyzer, String query, long branchId, long languageId, Date date, boolean searchName, boolean searchContent, boolean searchFields) throws QueryException;
 
     /**

trunk/daisy/repository/server/src/main/java/org/outerj/daisy/ftindex/FullTextIndexImpl.java

@@ -21 +21 @@
 import java.util.Arrays;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Lock;
@@ -67 +69 @@
 public class FullTextIndexImpl implements FullTextIndex, FullTextIndexImplMBean, SuspendableProcess {
     private static final int PRECISION_STEP_LONG = 6;
-    private Analyzer luceneAnalyzer;
-    private File indexDirectoryFile;
+    private String indexAnalyzerName;
+    private Analyzer indexAnalyzer;
+    private Analyzer fallbackAnalyzer = new StandardAnalyzer(Version.LUCENE_30);
+    private String defaultQueryAnalyzerName;
+    private Map<String, Analyzer> analyzersByName = new HashMap<String, Analyzer>();
+    private File indexDirectoryFile;
     private Directory indexDirectory;
     private IndexWriter indexWriter;
@@ -77 +83 @@
     private Thread indexFlushThread = null;
     private IndexOptimizeThread indexOptimizeThread = null;
+    private PluginUser<Analyzer> analyzerPluginUser = new AnalyzerPluginUser();
+    private boolean initDone = false;
     
     /**
@@ -115 +123 @@
     @PreDestroy
     public void destroy() throws Exception {
+        pluginRegistry.unsetPluginUser(Analyzer.class, analyzerPluginUser);
+
         this.stop();
         this.dispose();
@@ -127 +137 @@
             throw new ConfigurationException("The specified directory is not a directory: " + directoryName);
         log.debug("Using the following as directory to store indexes: " + indexDirectoryFile);
+        
+        indexAnalyzerName = configuration.getChild("indexAnalyzer").getValue(null);
+        defaultQueryAnalyzerName = configuration.getChild("defaultQueryAnalyzer").getValue(null);
 
         this.indexFlushInterval = configuration.getChild("indexFlushInterval").getValueAsInteger(indexFlushInterval);
@@ -146 +159 @@
         }
         
-        // set the analyzer
-        PluginUser<Analyzer> analyzerPluginUser = new AnalyzerPluginUser();
+        // register the plugin user
         pluginRegistry.setPluginUser(Analyzer.class, analyzerPluginUser);
-        pluginRegistry.unsetPluginUser(Analyzer.class, analyzerPluginUser);
-        if (luceneAnalyzer == null) {
-            luceneAnalyzer = new StandardAnalyzer(Version.LUCENE_30);
-        }
+        
+        // fetch the indexAnalyzer;
+        indexAnalyzer = getAnalyzer(indexAnalyzerName);
 
         // Make the initial IndexWriter instance
@@ -163 +174 @@
 
         pluginRegistry.addPlugin(SuspendableProcess.class, SUSPEND_PROCESS_NAME, this);
+        initDone = true;
     }
 
@@ -232 +244 @@
     }
 
-    public Hits search(String queryAsString, long branchId, long languageId, Date date, boolean searchName, boolean searchContent, boolean searchFields) throws QueryException {
+    public Hits search(String analyzerName, String queryAsString, long branchId, long languageId, Date date, boolean searchName, boolean searchContent, boolean searchFields) throws QueryException {
+        Analyzer analyzer;
+        if (analyzerName == null && defaultQueryAnalyzerName == null) {
+            analyzer = fallbackAnalyzer;
+        } else {
+            if (analyzerName == null) {
+                analyzerName = defaultQueryAnalyzerName;
+            }
+            if (!analyzersByName.containsKey(analyzerName)) {
+                throw new QueryException("No analyzer named " + analyzerName + " is registered");
+            }
+            analyzer = getAnalyzer(analyzerName);
+        }
         BooleanQuery query = new BooleanQuery();
         try {
@@ -244 +268 @@
 
             String[] strings = new String[fieldstoSearch.size()];
-            Arrays.fill(strings, queryAsString);
-            Query baseQuery = MultiFieldQueryParser.parse(Version.LUCENE_30, strings, fieldstoSearch.toArray(new String[fieldstoSearch.size()]), luceneAnalyzer);
+            Arrays.fill(strings, queryAsString);// analyzer = new org.apache.lucene.analysis.fr.FrenchAnalyzer(Version.LUCENE_30);
+            Query baseQuery = MultiFieldQueryParser.parse(Version.LUCENE_30, strings, fieldstoSearch.toArray(new String[fieldstoSearch.size()]), analyzer);
             
             query.add(baseQuery, BooleanClause.Occur.MUST);
@@ -422 +446 @@
 
     private IndexWriter constructIndexWriter() throws IOException {
-        return new IndexWriter(indexDirectory, luceneAnalyzer, MaxFieldLength.UNLIMITED);
+        return new IndexWriter(indexDirectory, indexAnalyzer, MaxFieldLength.UNLIMITED);
     }
 
@@ -530 +554 @@
     }
 
+    public Analyzer getAnalyzer(String name) {
+        Analyzer result = null; 
+        if (name != null) {
+            result = analyzersByName.get(name);
+        }
+        if (result == null) {
+            return fallbackAnalyzer;
+        }
+        return result;
+    }
+    
     private class AnalyzerPluginUser implements PluginUser<Analyzer> {
-        private boolean done = false;
-
         public void pluginAdded(PluginHandle<Analyzer> pluginHandle) {
-            if (!done) { // ignore all but the very first registered plugin
-                done = true;
-                FullTextIndexImpl.this.luceneAnalyzer = pluginHandle.getPlugin();
-                log.info("Using custom lucene analyzer registered with name " + pluginHandle.getName());
+            synchronized (analyzersByName) {
+                if (analyzersByName.containsKey(pluginHandle.getName())) {
+                    log.warn(String.format("Analyzer with name %s already registered, not registering it again", pluginHandle.getName()));
+                } else {
+                    log.info("Registering Lucene Analyzer with name " + pluginHandle.getName());
+                    analyzersByName.put(pluginHandle.getName(), pluginHandle.getPlugin());
+                    if (pluginHandle.getName().equals(indexAnalyzerName) && !FullTextIndexImpl.this.initDone) {
+                        log.error("Refusing to set the indexAnalyzer after starting the repository to avoid indexing errors");
+                    }
+                }
             }
         }
 
         public void pluginRemoved(PluginHandle<Analyzer> pluginHandle) {
-            // ignored
+            analyzersByName.remove(pluginHandle.getName());
         }
     }

trunk/daisy/repository/server/src/main/java/org/outerj/daisy/query/model/Query.java

@@ -60 +60 @@
     private boolean annotateLinkFields = true;
     private String styleHint;
+    private String analyzerName;
     private SqlGenerationContext sqlGenerationContext;
     private int chunkOffset = 1;
@@ -222 +223 @@
                 throw new NumberFormatException("The following value could not be used as a chunk length : " + value + ". Chunk lengths must be numeric.");
             }
+        } else if (name.equalsIgnoreCase("ft_query_analyzer")) {
+          this.analyzerName = value;  
         } else
             throw new RuntimeException("Unrecognized option: " + name);
@@ -280 +283 @@
         this.chunkOffset = chunkOffset;
     }
+    
+    public String getAnalyzerName() {
+        return analyzerName;
+    }
 
     /**

trunk/daisy/repository/server/src/main/java/org/outerj/daisy/repository/serverimpl/query/LocalQueryManager.java

@@ -491 +491 @@
                     pointInTime = query.getVersionMode().getDate();
                 }
-                fullTextHits = fullTextIndex.search(ftQuery.getQuery(), ftQuery.getBranchId(), ftQuery.getLanguageId(), pointInTime, 
+                fullTextHits = fullTextIndex.search(query.getAnalyzerName(), ftQuery.getQuery(), ftQuery.getBranchId(), ftQuery.getLanguageId(), pointInTime, 
                         ftQuery.getSearchName(), ftQuery.getSearchContent(), ftQuery.getSearchFields());
                 evaluationInfo.setHits(fullTextHits);