DYN-5806 lucene node autocomplete (#14169)

RobertGlobant20 · web-flow · commit 380bd8e408c8 · 2023-07-21T17:42:07.000-04:00
* DYN-5806 Lucene NodeAutocomplete

Added functionality for enabling the use of Lucene Search in node autocomplete, also I've modified the LuceneSearchUtility class for supporting in-memory indexing.

* DYN-5806-Lucene-NodeAutocomplete

Changes for indexing (in memory RAMDirectory) a subset of nodes every time that the node autocomplete search functionality is executed so the search will be executed using only the subset of nodes indexed (and not over all the 3000 nodes).

* DYN-5806-Lucene-NodeAutocomplete Code Review

Adding brackets in else statement and removing the Distinct() due that all the nodes should be different then there is no need to use this function

* DYN-5806-Lucene-NodeAutocomplete Code Review 2

Rolling back the code for SearchViewModel.Search() due that tests are using this method (Lucene indexing is not done when tests are executed - IsTestMode = true).
Also I've modified the part in which Node Autocomplete is initialized so it will calling the Lucene search to initialize the results.

* DYN-5806-Lucene-NodeAutocomplete Code Review 2

Removing unnecessary Distinct()

* DYN-5806-Lucene-NodeAutocomplete Regressions

Trying to fix regressions generated when trying to initialize NodeAutocomplete with Lucene Search.

* DYN-5806-Lucene-NodeAutocomplete Regressions

I've added an extra validation for checking if the StorageType is RAMDirectory or FSDirectory otherwise it won't do things like create the indexWriter or get a new Lucene Document (due that previously were skipped if IsTestMode is True).

* DYN-5806-Lucene-NodeAutocomplete Regressions

Fixing comment

* DYN-5806-Lucene-NodeAutocomplete Regressions

re-triggering pullRequestValidation job
diff --git a/src/DynamoCore/Utilities/LuceneSearchUtility.cs b/src/DynamoCore/Utilities/LuceneSearchUtility.cs
@@ -21,6 +21,16 @@ internal class LuceneSearchUtility
         internal Lucene.Net.Store.Directory indexDir;
         internal IndexWriter writer;
         internal string directory;
+        internal LuceneStorage currentStorageType;
+
+        public enum LuceneStorage
+        {
+            //Lucene Storage will be located in RAM and all the info indexed will be lost when Dynamo app is closed
+            RAM,
+
+            //Lucene Storage will be located in the local File System and the files will remain in ...AppData\Roaming\Dynamo\Dynamo Core\2.19\Index folder
+            FILE_SYSTEM
+        }
 
         // Used for creating the StandardAnalyzer
         internal Analyzer Analyzer;
@@ -36,23 +46,34 @@ internal LuceneSearchUtility(DynamoModel model)
         /// <summary>
         /// Initialize Lucene config file writer.
         /// </summary>
-        internal void InitializeLuceneConfig(string dirName)
+        internal void InitializeLuceneConfig(string dirName, LuceneStorage storageType = LuceneStorage.FILE_SYSTEM)
         {
             addedFields = new List<string>();
 
-            DirectoryInfo webBrowserUserDataFolder;
+            DirectoryInfo luceneUserDataFolder;
             var userDataDir = new DirectoryInfo(dynamoModel.PathManager.UserDataDirectory);
-            webBrowserUserDataFolder = userDataDir.Exists ? userDataDir : null;
+            luceneUserDataFolder = userDataDir.Exists ? userDataDir : null;
 
             directory = dirName;
-            string indexPath = Path.Combine(webBrowserUserDataFolder.FullName, LuceneConfig.Index, dirName);
-            indexDir = Lucene.Net.Store.FSDirectory.Open(indexPath);
+            string indexPath = Path.Combine(luceneUserDataFolder.FullName, LuceneConfig.Index, dirName);
+
+            currentStorageType = storageType;
+
+            if (storageType == LuceneStorage.RAM)
+            {
+                indexDir = new RAMDirectory();
+            }
+            else
+            {
+                indexDir = FSDirectory.Open(indexPath);
+            }
+                
 
             // Create an analyzer to process the text
             Analyzer = new StandardAnalyzer(LuceneConfig.LuceneNetVersion);
 
-            // Initialize Lucene index writer, unless in test mode.
-            if (!DynamoModel.IsTestMode)
+            // Initialize Lucene index writer, unless in test mode or we are using RAMDirectory for indexing info. 
+            if (!DynamoModel.IsTestMode || currentStorageType == LuceneStorage.RAM)
             {
                 // Create an index writer
                 IndexWriterConfig indexConfig = new IndexWriterConfig(LuceneConfig.LuceneNetVersion, Analyzer)
@@ -77,7 +98,7 @@ internal void InitializeLuceneConfig(string dirName)
         /// <returns></returns>
         internal Document InitializeIndexDocumentForNodes()
         {
-            if (DynamoModel.IsTestMode) return null;
+            if (DynamoModel.IsTestMode && currentStorageType == LuceneStorage.FILE_SYSTEM) return null;
 
             var name = new TextField(nameof(LuceneConfig.NodeFieldsEnum.Name), string.Empty, Field.Store.YES);
             var fullCategory = new TextField(nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), string.Empty, Field.Store.YES);
@@ -153,7 +174,7 @@ internal void SetDocumentFieldValue(Document doc, string field, string value, bo
                 ((StringField)doc.GetField(field)).SetStringValue(value);
             }
 
-            if (isLast && indexedFields.Any())
+            if (isLast && indexedFields != null && indexedFields.Any())
             {
                 List<string> diff = indexedFields.Except(addedFields).ToList();
                 foreach (var d in diff)
@@ -248,7 +269,7 @@ internal string CreateSearchQuery(string[] fields, string SearchTerm)
         internal void DisposeWriter()
         {
             //We need to check if we are not running Dynamo tests because otherwise parallel test start to fail when trying to write in the same Lucene directory location
-            if (!DynamoModel.IsTestMode)
+            if (!DynamoModel.IsTestMode || currentStorageType == LuceneStorage.RAM)
             {
                 writer?.Dispose();
                 writer = null;
@@ -257,7 +278,7 @@ internal void DisposeWriter()
 
         internal void CommitWriterChanges()
         {
-            if (!DynamoModel.IsTestMode)
+            if (!DynamoModel.IsTestMode || currentStorageType == LuceneStorage.RAM)
             {
                 //Commit the info indexed
                 writer?.Commit();
diff --git a/src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs
@@ -18,6 +18,9 @@
 using Dynamo.Utilities;
 using Dynamo.Wpf.ViewModels;
 using Greg;
+using Lucene.Net.Documents;
+using Lucene.Net.QueryParsers.Classic;
+using Lucene.Net.Search;
 using Newtonsoft.Json;
 using ProtoCore.AST.AssociativeAST;
 using ProtoCore.Mirror;
@@ -39,6 +42,9 @@ public class NodeAutoCompleteSearchViewModel : SearchViewModel
         private bool displayLowConfidence;
         private const string nodeAutocompleteMLEndpoint = "MLNodeAutocomplete";
 
+        // Lucene search utility to perform indexing operations just for NodeAutocomplete.
+        internal LuceneSearchUtility LuceneSearchUtilityNodeAutocomplete { get; set; }
+
         /// <summary>
         /// The Node AutoComplete ML service version, this could be empty if user has not used ML way
         /// </summary>
@@ -602,6 +608,61 @@ private NodeSearchElementViewModel GetViewModelForNodeSearchElement(NodeSearchEl
             return null;
         }
 
+
+        /// <summary>
+        ///     Performs a search using the given string as query and subset, if provided.
+        /// </summary>
+        /// <returns> Returns a list with a maximum MaxNumSearchResults elements.</returns>
+        /// <param name="search"> The search query </param>
+        /// <param name="useLucene"> Temporary flag that will be used for searching using Lucene.NET </param>
+        internal IEnumerable<NodeSearchElementViewModel> SearchNodeAutocomplete(string search, bool useLucene)
+        {
+            if (useLucene)
+            {
+                //The DirectoryReader and IndexSearcher have to be assigned after commiting indexing changes and before executing the Searcher.Search() method, otherwise new indexed info won't be reflected
+                LuceneSearchUtilityNodeAutocomplete.dirReader = LuceneSearchUtilityNodeAutocomplete.writer?.GetReader(applyAllDeletes: true);
+                if (LuceneSearchUtilityNodeAutocomplete.dirReader == null) return null;
+
+                LuceneSearchUtilityNodeAutocomplete.Searcher = new IndexSearcher(LuceneSearchUtilityNodeAutocomplete.dirReader);
+
+                string searchTerm = search.Trim();
+                var candidates = new List<NodeSearchElementViewModel>();
+                var parser = new MultiFieldQueryParser(LuceneConfig.LuceneNetVersion, LuceneConfig.NodeIndexFields, LuceneSearchUtilityNodeAutocomplete.Analyzer)
+                {
+                    AllowLeadingWildcard = true,
+                    DefaultOperator = LuceneConfig.DefaultOperator,
+                    FuzzyMinSim = LuceneConfig.MinimumSimilarity
+                };
+
+                Query query = parser.Parse(LuceneSearchUtilityNodeAutocomplete.CreateSearchQuery(LuceneConfig.NodeIndexFields, searchTerm));
+                TopDocs topDocs = LuceneSearchUtilityNodeAutocomplete.Searcher.Search(query, n: LuceneConfig.DefaultResultsCount);
+
+                for (int i = 0; i < topDocs.ScoreDocs.Length; i++)
+                {
+                    // read back a Lucene doc from results
+                    Document resultDoc = LuceneSearchUtilityNodeAutocomplete.Searcher.Doc(topDocs.ScoreDocs[i].Doc);
+
+                    string name = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.Name));
+                    string docName = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.DocName));
+                    string cat = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName));
+                    string parameters = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.Parameters));
+
+
+                    var foundNode = FindViewModelForNodeNameAndCategory(name, cat, parameters);
+                    if (foundNode != null)
+                    {
+                        candidates.Add(foundNode);
+                    }
+                }
+
+                return candidates;
+            }
+            else
+            {
+                return Search(search);
+            }
+        }
+
         /// <summary>
         /// Filters the matching node search elements based on user input in the search field. 
         /// </summary>
@@ -617,9 +678,25 @@ internal void SearchAutoCompleteCandidates(string input)
                 }
                 else
                 {
-                    // Providing the saved search results to limit the scope of the query search.
-                    // Then add back the ML info on filterted nodes as the Search function accepts elements of type NodeSearchElement
-                    var foundNodes = Search(input, searchElementsCache.Select(x => x.Model));
+                    LuceneSearchUtilityNodeAutocomplete = new LuceneSearchUtility(dynamoViewModel.Model);
+
+                    //The dirName parameter doesn't matter because we are using RAMDirectory indexing and no files are created
+                    LuceneSearchUtilityNodeAutocomplete.InitializeLuceneConfig(string.Empty, LuceneSearchUtility.LuceneStorage.RAM);
+
+                    //Memory indexing process for Node Autocomplete (indexing just the nodes returned by the NodeAutocomplete service so we limit the scope of the query search)
+                    foreach (var node in searchElementsCache.Select(x => x.Model))
+                    {
+                        var doc = LuceneSearchUtilityNodeAutocomplete.InitializeIndexDocumentForNodes();
+                        AddNodeTypeToSearchIndex(node, doc);
+                    }
+
+                    //Write the Lucene documents to memory
+                    LuceneSearchUtilityNodeAutocomplete.CommitWriterChanges();
+                        
+                    var luceneResults = SearchNodeAutocomplete(input, true);
+                    var foundNodesModels = luceneResults.Select(x => x.Model);
+                    var foundNodes = foundNodesModels.Select(MakeNodeSearchElementVM);
+
                     var filteredSearchElements = new List<NodeSearchElementViewModel>();
 
                     foreach (var node in foundNodes)
@@ -635,10 +712,30 @@ internal void SearchAutoCompleteCandidates(string input)
                         }
                     }
                     FilteredResults = new List<NodeSearchElementViewModel>(filteredSearchElements).OrderBy(x => x.Name).ThenBy(x => x.Description);
+
+                    LuceneSearchUtilityNodeAutocomplete.DisposeWriter();
                 }
             }
         }
 
+        /// <summary>
+        /// Add node information to Lucene index
+        /// </summary>
+        /// <param name="node">node info that will be indexed</param>
+        /// <param name="doc">Lucene document in which the node info will be indexed</param>
+        private void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc)
+        {
+            if (LuceneSearchUtilityNodeAutocomplete.addedFields == null) return;
+
+            LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), node.FullCategoryName);
+            LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Name), node.Name);
+            LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Description), node.Description);
+            if (node.SearchKeywords.Count > 0) LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.SearchKeywords), node.SearchKeywords.Aggregate((x, y) => x + " " + y), true, true);
+            LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Parameters), node.Parameters ?? string.Empty);
+
+            LuceneSearchUtilityNodeAutocomplete.writer?.AddDocument(doc);
+        }
+
         /// <summary>
         /// Returns a collection of node search elements for nodes
         /// that output a type compatible with the port type if it's an input port.
diff --git a/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs
@@ -995,7 +995,7 @@ internal IEnumerable<NodeSearchElementViewModel> Search(string search, bool useL
         /// <param name="nodeCategory">Full Category of the node</param>
         /// <param name="parameters">Node input parameters</param>
         /// <returns></returns>
-        private NodeSearchElementViewModel FindViewModelForNodeNameAndCategory(string nodeName, string nodeCategory, string parameters)
+        internal NodeSearchElementViewModel FindViewModelForNodeNameAndCategory(string nodeName, string nodeCategory, string parameters)
         {
             var result = Model.SearchEntries.Where(e => {
                 if (e.Name.Equals(nodeName) && e.FullCategoryName.Equals(nodeCategory))
@@ -1034,7 +1034,7 @@ private static IEnumerable<NodeSearchElementViewModel> GetVisibleSearchResults(N
             }
         }
 
-        private NodeSearchElementViewModel MakeNodeSearchElementVM(NodeSearchElement entry)
+        internal NodeSearchElementViewModel MakeNodeSearchElementVM(NodeSearchElement entry)
         {
             var element = entry as CustomNodeSearchElement;
             var elementVM = element != null

Original file line number	Diff line number	Diff line change
`@@ -995,7 +995,7 @@ internal IEnumerable<NodeSearchElementViewModel> Search(string search, bool useL`
`995`	`995`	`/// <param name="nodeCategory">Full Category of the node</param>`
`996`	`996`	`/// <param name="parameters">Node input parameters</param>`
`997`	`997`	`/// <returns></returns>`
`998`		`- private NodeSearchElementViewModel FindViewModelForNodeNameAndCategory(string nodeName, string nodeCategory, string parameters)`
	`998`	`+ internal NodeSearchElementViewModel FindViewModelForNodeNameAndCategory(string nodeName, string nodeCategory, string parameters)`
`999`	`999`	`{`
`1000`	`1000`	`var result = Model.SearchEntries.Where(e => {`
`1001`	`1001`	`if (e.Name.Equals(nodeName) && e.FullCategoryName.Equals(nodeCategory))`
`@@ -1034,7 +1034,7 @@ private static IEnumerable<NodeSearchElementViewModel> GetVisibleSearchResults(N`
`1034`	`1034`	`}`
`1035`	`1035`	`}`
`1036`	`1036`
`1037`		`- private NodeSearchElementViewModel MakeNodeSearchElementVM(NodeSearchElement entry)`
	`1037`	`+ internal NodeSearchElementViewModel MakeNodeSearchElementVM(NodeSearchElement entry)`
`1038`	`1038`	`{`
`1039`	`1039`	`var element = entry as CustomNodeSearchElement;`
`1040`	`1040`	`var elementVM = element != null`