LU fixes (#657)

* bug fixes

* fixing tests

Co-authored-by: Vishwac Sena Kannan <vishwacsenakannan@WIN-9GFQV4IK7VB.redmond.corp.microsoft.com>

Author: vishwacsena

packages/lu/src/parser/lufile/parseFileContents.js

@@ -1034,7 +1034,7 @@ const parseAndHandleEntityV2 = function (parsedContent, luResource, log, locale)
         for (const entity of entities) {
             if (entity.Type !== INTENTTYPE) {
                 entity.Name = entity.Name.replace(/^[\'\"]|[\'\"]$/g, "");
-                let entityName = entity.Name;
+                let entityName = entity.Name.endsWith('=') ? entity.Name.slice(0, entity.Name.length - 1) : entity.Name;
                 let entityType = !entity.Type ? getEntityType(entity.Name, entities) : entity.Type;
                 if (!entityType) {
                     let errorMsg = `No type definition found for entity "${entityName}". Supported types are ${Object.values(EntityTypeEnum).join(', ')}. Note: Type names are case sensitive.`;

packages/lu/src/parser/luis/luisCollate.js

@@ -52,6 +52,7 @@ const collate = function(luisList) {
         buildPatternAny(blob, luisObject)
     }
     helpers.checkAndUpdateVersion(luisObject)
+    cleanupEntities(luisObject)
     return luisObject
 }
 
@@ -60,6 +61,19 @@ module.exports = {
     build
 }
 
+const cleanupEntities = function(luisObject) {
+    let consolidatedList = [];
+    luisObject.composites.forEach(item => consolidatedList.push(item));
+    luisObject.closedLists.forEach(item => consolidatedList.push(item));
+    luisObject.regex_entities.forEach(item => consolidatedList.push(item));
+    luisObject.prebuiltEntities.forEach(item => consolidatedList.push(item));
+    let idxToRemove = [];
+    luisObject.entities.forEach((item, idx) => {
+        if (consolidatedList.find(e => e.name == item.name) !== undefined) idxToRemove.push(idx);
+    })
+    idxToRemove.sort((a, b) => a-b).forEach(idx => luisObject.entities.splice(idx, 1))
+}
+
 const mergeResultsWithHash = function (blob, finalCollection, type, hashTable) {
     if (blob[type].length === 0) { 
         return

packages/lu/test/parser/lufile/parseFileContents.NewEntityDefinition.test.js

@@ -3,6 +3,8 @@
  * Licensed under the MIT License.
  */
 const parseFile = require('../../../src/parser/lufile/parseFileContents');
+const luObj = require('./../../../src/parser/lu/lu');
+const luBuilder = require('./../../../src/parser/luis/luisBuilder');
 var chai = require('chai');
 var assert = chai.assert;
 describe('V2 Entity definitions using @ notation', function () {
@@ -1080,6 +1082,21 @@ describe('V2 Entity definitions using @ notation', function () {
                 .then(res => done(res))
                 .catch(err => done())
         });
+
+        it('BF CLI #653 - closed list definition', function(done){
+            let luFile = `
+            @ list blah=
+                - something:
+            `;
+
+            parseFile.parseFile(luFile)
+                .then(res => {
+                    assert.equal(res.LUISJsonStructure.closedLists[0].name, "blah");
+                    assert.equal(res.LUISJsonStructure.closedLists[0].subLists[0].canonicalForm, "something");
+                    done()
+                })
+                .catch(err => done(err))
+        })
     });
 
     describe('Pattern.Any entity definition', function(){
@@ -1113,5 +1130,24 @@ describe('V2 Entity definitions using @ notation', function () {
                 .catch(err => done(err));
         })
     })
+
+    describe('multi-content', function(){
+        it('BF-CLI #652 - multi entity definition across docs is merged correctly', function(done){
+            let luContent1 = new luObj(`@ list blah=
+            - something:
+            
+            @ composite foo = [blah]
+            `);
+            let luContent2 = new luObj(`# utterances
+            - @{foo= something}`);
+            luBuilder.fromLUAsync([luContent1, luContent2])
+                .then(res => {
+                    assert.isTrue(res.entities.length === 0);
+                    assert.equal(res.composites[0].name, "foo");
+                    done()
+                })
+                .catch(err => done(err))
+        })
+    })
 
 });

packages/lu/test/parser/lufile/parseFileContents.regexEntity.test.js

@@ -137,8 +137,7 @@ $test:/hrf-[0-9]{6}/`;
 
     it('throws when duplicate regex entities with different patterns are found across lu files', function(done) {
         let luFile1 = `$test:/hrf-[0-9]{6}/`;
-        let luFile2 = `# test
-- this is a {test=one} utterance`;
+        let luFile2 = `$test:/bpg-[0-9]{5}/`;
         parseFile(luFile1, false) 
             .then(res1 => {
                 parseFile(luFile2, false)