I am fairly new to ElasticSearch and NEST, and i have run into an issue.
I am in the middle of trying to add analyzers and tokenizers, so that I can search substrings in my code.
Example:
User user1 = new User(){ FirstName = "John", LastName = "Boat", Number="45678" }
User user2 = new User(){ FirstName = "Michael", LastName = "Johansen", Number="123456" }
Searching "12345" yields user2, "456" yields user1 & user2, "Joh" yields user1 & user2, etc.
However when trying to add analyzers and tokenfilters to my settings when creating the index, they are not getting saved in the elastic database.
This works:
client.Indices.Create("customers",
index => index
.Settings(se => se
.Setting("index.mapping.total_fields.limit", "2000"))
.Map<Customer>(x => x.AutoMap())
);
This does not work:
client.Indices.Create("crmleads",
index => index
.Settings(se => se
.Analysis(a => a
.Analyzers(analyzer => analyzer
.Custom("substring_analyzer", analyzerDescriptor => analyzerDescriptor
.Tokenizer("standard")
.Filters("lowercase", "substring")))
.TokenFilters(tf => tf
.NGram("substring", filterDescriptor => filterDescriptor
.MinGram(2)
.MaxGram(15))))
.Setting("index.mapping.total_fields.limit", "2000"))
.Map<CRMLead>(x => x
.AutoMap()
.Properties(p => p
.Text(t => t
.Name(f => f.Name)
.Analyzer("substring_analyzer"))
.Text(t => t
.Name(f => f.CVRNumber)
.Analyzer("substring_analyzer"))
.Boolean(t => t
.Name(f => f.IsConverted))
.Text(t => t
.Name(f => f.ContactPersonName)
.Analyzer("substring_analyzer"))
.Text(t => t
.Name(f => f.ContactPersonEmail
).Analyzer("substring_analyzer"))))
);
Also the command line for the ElasticSearch server does not show any errors:
Command line for ElasticSearch
Created indexes shown in Kibana
Also i have not added anything to my model classes:
public class CRMLead
{
[Key]
[DatabaseGenerated(DatabaseGeneratedOption.Identity)]
public Guid Id { get; set; }
public Company Company { get; set; }
public Customer Customer { get; set; }
public string CVRNumber { get; set; }
public DateTime CreateDate { get; set; }
public string Name { get; set; }
public string Website { get; set; }
public string Country { get; set; }
public string Address { get; set; }
public string ZipCode { get; set; }
public string City { get; set; }
public string ContactPersonName { get; set; }
public string ContactPersonEmail { get; set; }
public string ContactPersonPhoneNumber { get; set; }
public string PhaseOneDescription { get; set; }
public CustomerContact CustomerContact { get; set; }
public ApplicationUser Seller { get; set; }
public CRMLeadStatus CRMStatus { get; set; }
public List<UploadedFile> UploadedFiles { get; set; }
public bool IsConverted { get; set; }
public bool IsDone { get; set; }
public bool IsSold { get; set; }
}
Full Code here:
public static void AddElasticsearch(this IServiceCollection services, IConfiguration configuration)
{
var url = configuration["Elasticsearch:url"];
var settings = new ConnectionSettings(new Uri(url));
AddDefaultMappings(settings);
settings.DefaultFieldNameInferrer(f => f);
var client = new ElasticClient(settings);
services.AddSingleton<IElasticClient>(client);
CreateIndices(client);
}
private static void AddDefaultMappings(ConnectionSettings settings)
{
settings
.DefaultMappingFor<ApplicationUser>(m => m
.IndexName("users")
.Ignore(au => au.AccessFailedCount)
.Ignore(au => au.Address)
.Ignore(au => au.AppInstalled)
.Ignore(au => au.BirthDay)
.Ignore(au => au.BorrowedEquipment)
.Ignore(au => au.ConcurrencyStamp)
.Ignore(au => au.CostPrice)
.Ignore(au => au.Culture)
.Ignore(au => au.CustomUserfields)
.Ignore(au => au.EmailConfirmed)
.Ignore(au => au.HireDate)
.Ignore(au => au.IceRelatives)
.Ignore(au => au.Id)
.Ignore(au => au.Initials)
.Ignore(au => au.IsCompanyOwner)
.Ignore(au => au.LastLogin)
.Ignore(au => au.LockoutEnabled)
.Ignore(au => au.LockoutEnd)
.Ignore(au => au.NormalizedEmail)
.Ignore(au => au.NormalizedUserName)
.Ignore(au => au.PasswordHash)
.Ignore(au => au.PhoneNumberConfirmed)
.Ignore(au => au.PrivateEmail)
.Ignore(au => au.PrivatePhoneNumber)
.Ignore(au => au.ProfileImagePath)
.Ignore(au => au.SecurityStamp)
.Ignore(au => au.TwoFactorEnabled)
.Ignore(au => au.UploadedFile)
)
.DefaultMappingFor<CRMLead>(m => m
.IndexName("crmleads")
.Ignore(crml => crml.Address)
.Ignore(crml => crml.City)
.Ignore(crml => crml.Company)
.Ignore(crml => crml.ContactPersonPhoneNumber)
.Ignore(crml => crml.Country)
.Ignore(crml => crml.CreateDate)
.Ignore(crml => crml.Customer)
.Ignore(crml => crml.CustomerContact)
.Ignore(crml => crml.IsDone)
.Ignore(crml => crml.IsSold)
.Ignore(crml => crml.PhaseOneDescription)
.Ignore(crml => crml.Seller)
.Ignore(crml => crml.UploadedFiles)
.Ignore(crml => crml.Website)
.Ignore(crml => crml.ZipCode)
)
.DefaultMappingFor<Customer>(m => m
.IndexName("customers")
.Ignore(cust => cust.Activities)
.Ignore(cust => cust.Address)
.Ignore(cust => cust.City)
.Ignore(cust => cust.Company)
.Ignore(cust => cust.Contacts)
.Ignore(cust => cust.Country)
.Ignore(cust => cust.CreateDate)
.Ignore(cust => cust.Id)
.Ignore(cust => cust.Projects)
.Ignore(cust => cust.ZipCode)
)
}
private static void CreateIndices(IElasticClient client)
{
client.Indices.Create("users",
index => index
.Map<ApplicationUser>(x => x.AutoMap())
);
client.Indices.Create("crmleads",
index => index
.Settings(se => se
.Analysis(a => a
.Analyzers(analyzer => analyzer
.Custom("substring_analyzer", analyzerDescriptor => analyzerDescriptor
.Tokenizer("standard")
.Filters("lowercase", "substring")))
.TokenFilters(tf => tf
.NGram("substring", filterDescriptor => filterDescriptor
.MinGram(2)
.MaxGram(15))))
.Setting("index.mapping.total_fields.limit", "2000"))
.Map<CRMLead>(x => x
.AutoMap()
.Properties(p => p
.Text(t => t
.Name(f => f.Name)
.Analyzer("substring_analyzer"))
.Text(t => t
.Name(f => f.CVRNumber)
.Analyzer("substring_analyzer"))
.Boolean(t => t
.Name(f => f.IsConverted))
.Text(t => t
.Name(f => f.ContactPersonName)
.Analyzer("substring_analyzer"))
.Text(t => t
.Name(f => f.ContactPersonEmail
).Analyzer("substring_analyzer"))))
);
client.Indices.Create("customers",
index => index
.Settings(se => se
.Setting("index.mapping.total_fields.limit", "2000"))
.Map<Customer>(x => x.AutoMap())
);
}
Technologies are:
What am I doing wrong here? Thanks in advance.
EDIT
After studying the index response (as @Milan Gatyas suggested) i found out that my tokenfilter NGram.MaxGram expected a difference of 1 but was 13
.NGram("substring", filterDescriptor => filterDescriptor
.MinGram(2)
.MaxGram(15))))
So to fix this I set the setting for max_ngram_diff to 15:
.Setting("index.max_ngram_diff", "15"))
To fix this i set the setting for max_ngram_diff to 15:
.Setting("index.max_ngram_diff", "15"))