需要使用C#和Nest从ElasticSearch返回没有完整搜索结果文档的不同字段

Question

我的日志存储在ElasticSearch和一个使用C和Nest的Windows应用程序中，Nest正在对ElasticSearch执行搜索。ElasticSearch中的映射如下所示：

"mappings": {
    "qns": {
        "properties": {
            "@timestamp": {
                "format": "strict_date_optional_time||epoch_millis",
                "type": "date"
            },
            "Error_Description": {
                "index": "not_analyzed",
                "type": "string"
            },
            "Thread_Id": {
                "index": "not_analyzed",
                "type": "string"
            },
            "Error_Description_Analyzed": {
                "type": "string"
            },
            "Error_Source": {
                "index": "not_analyzed",
                "type": "string"
            },
            "record": {
                "type": "string"
            },
            "@version": {
                "type": "string"
            },
            "Log_Level": {
                "type": "string"
            },
            "Record": {
                "type": "string"
            },
            "id": {
                "type": "long"
            },
            "Error_Source_Analyzed": {
                "type": "string"
            },
            "Timestamp": {
                "format": "strict_date_optional_time||epoch_millis",
                "type": "date"
            }
        }
    }
}

相应的C类如下：

[ElasticsearchType(IdProperty = "Id", Name = "qns")]
public class QNS
{
    [Number(NumberType.Long, Name = "id")]
    public long Id { get; set; }

    [Date(Name = "Timestamp")]
    public DateTime Timestamp { get; set; }

    [String(Name = "Error_Description", Index = FieldIndexOption.NotAnalyzed)]
    public string ErrorDescriptionKeyword { get; set; }

    [String(Name = "Error_Description_Analyzed")]
    public string ErrorDescriptionAnalyzed { get; set; }

    [String(Name = "Error_Source", Index = FieldIndexOption.NotAnalyzed)]
    public string ErrorSourceKeyword { get; set; }

    [String(Name = "Error_Source_Analyzed")]
    public string ErrorSourceAnalyzed { get; set; }

    [String(Name = "Thread_Id", Index = FieldIndexOption.NotAnalyzed)]
    public string ThreadId { get; set; }

    [String(Name = "Log_Level")]
    public string LogLevel { get; set; }

    [String(Index = FieldIndexOption.NotAnalyzed)]
    public string Record { get; set; }
}

我需要一种方法来搜索在datetime范围内并匹配特定模式范围的不同错误记录。虽然我能够得到结果，但我也得到了所有满足搜索的文档，而我只需要不同的错误字符串。对于Distinct查询，我使用FluentNest(https://github.com/hoonzis/fluentnest). 检索结果的代码如下：

    private List<string> FindDistinctErrorsByPatternAndTimeRangeInternal(DateTime fromDateTime, DateTime toDateTime, List<pattern> patterns, string indexName, string type)
    {
        var documents = new List<QNS>();

        var fromTime = fromDateTime.ToString(Constants.IndexSearch.ES_DATETIME_FORMAT);
        var toTime = toDateTime.ToString(Constants.IndexSearch.ES_DATETIME_FORMAT);

        var patternQueries = new List<QueryContainer>();

        foreach (var p in patterns)
        {
            var pType = PatternType.unknown;
            if (Enum.TryParse<PatternType>(p.Pattern_Type.ToLowerInvariant(), out pType))
            {
                switch (pType)
                {
                    case PatternType.word:
                        patternQueries.Add(Query<QNS>.Regexp(r =>
                            r.Field(f =>
                                f.ErrorDescriptionAnalyzed)
                                .Value(p.Pattern_Description)
                            )
                       );
                        break;
                    case PatternType.phrase:
                        patternQueries.Add(Query<QNS>.MatchPhrase(m =>
                            m.Field(f =>
                                f.ErrorDescriptionAnalyzed)
                                .Query(p.Pattern_Description)
                            )
                        );
                        break;
                    case PatternType.unknown:
                    default:
                        break;
                }
            }
        }

        var datetimeQuery = Query<QNS>.QueryString(q =>
                                q.DefaultField(f =>
                                    f.Timestamp).Query($"[{fromTime} TO {toTime}]")
                                );

        var searchResults = client.Search<QNS>(s => s.Index(indexName)
           .Type(type)
           .Query(q =>
               q.Filtered(f =>
                   f.Filter(fq =>
                       fq.Bool(b =>
                           b.MinimumShouldMatch(1).Should(patternQueries.ToArray())
                       )
                   )
                   .Query(qd =>
                       qd.Bool(b =>
                           b.Must(datetimeQuery)
                       )
                   )
               )
            )
           .Sort(sort => sort.Ascending(SortSpecialField.DocumentIndexOrder))
           .Aggregations(agg => agg.DistinctBy(q => q.ErrorDescriptionKeyword)));

        var results = searchResults.Aggs.AsContainer<QNS>().GetDistinct(d => d.ErrorDescriptionKeyword);

        return results.ToList();
    }

我需要修改此代码以仅返回不同的错误字符串，而不是整个结果集。查询中的命中视频数约为3500，仅存在2个不同的错误字符串。因此，取回所有这些记录是没有意义的，因为我不打算使用它。有人能帮助我使用日期范围和模式正则表达式/短语匹配来获得正确的聚合查询，以便仅使用Nest或Nest/FluentNest返回不同的错误记录。

Answer 1

我想你正在寻找术语聚合。

但你的整个查询有点奇怪。您是否有一些遗留要求。

首先，您有两个字段ErrorDescriptionAnalyked和ErrorDescriptionKeyword您是否创建了一个不同的字段只是为了分析一个而不是一个？为什么不使用多字段

第二种过滤方法已经过时一段时间了。

这是一个快速示例，我希望能有所帮助

ElasticClient db = new ElasticClient(uri);

            db.DeleteIndex(indexName);

            var mappings = new CreateIndexDescriptor(indexName).Mappings(ms => ms.Map<A>(map => map.
                AutoMap().
                Properties(props =>
                    props.String(p =>
                        p.Name(a => a.Text).
                        Fields(fields =>
                            fields.String(pr => pr.Name("raw").NotAnalyzed()))))));

            db.CreateIndex(mappings);

            foreach (var item in Enumerable.Range(0, 10).Select(i => new A
            {
                Price1 = random.NextDouble() * 1000,
                Date = i % 3 == 0 ? new DateTime(1900, 1, 1) : DateTime.Now,
                Text = i % 2 == 0 ? "ABC" : "EFG"
            }))
            {
                db.Index(item, inx => inx.Index(indexName));
            }

            var toDate = DateTime.Now + TimeSpan.FromDays(1);
            var fromDate = DateTime.Now - TimeSpan.FromDays(30);

            var data = db.Search<A>(s => 
                s.Index(indexName)
                .Query(q=>
                        q.DateRange(r => r.Field(f => f.Date).GreaterThan(fromDate).LessThanOrEquals(toDate))
                        &&
                        (
                        //term query is for finding words by default all words are lowercase but you can set a different analyzer
                        q.Term(t => t.Field(f => f.Text).Value("ABC".ToLower()))
                        ||
//Raw field is not analysed so no need to lower case you can use you query here if you want
                        q.Term(t => t.Field("text.raw").Value("EFG"))
                        )
                ).Aggregations(aggr => aggr.Terms("distinct", aterm => aterm.Field("text.raw"))));