Browse Source
Support elastic search for code search (#10273)
* Support elastic search for code search * Finished elastic search implementation and add some tests * Enable test on drone and added docs * Add new fields to elastic search * Fix bug * remove unused changes * Use indexer alias to keep the gitea indexer version * Improve codes * Some code improvements * The real indexer name changed to xxx.v1 Co-authored-by: zeripath <art27@cantab.net>for-closed-social
Lunny Xiao
4 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 694 additions and 164 deletions
Unified View
Diff Options
-
+1 -0.drone.yml
-
+8 -0custom/conf/app.example.ini
-
+4 -0docs/content/doc/advanced/config-cheat-sheet.en-us.md
-
+5 -1docs/content/doc/advanced/config-cheat-sheet.zh-cn.md
-
+51 -81modules/indexer/code/bleve.go
-
+3 -50modules/indexer/code/bleve_test.go
-
+385 -0modules/indexer/code/elastic_search.go
-
+36 -0modules/indexer/code/elastic_search_test.go
-
+71 -19modules/indexer/code/indexer.go
-
+83 -0modules/indexer/code/indexer_test.go
-
+25 -5modules/indexer/code/queue.go
-
+4 -2modules/indexer/code/wrapped.go
-
+12 -0modules/setting/indexer.go
-
+6 -6templates/explore/code.tmpl
+ 1
- 0
.drone.yml
View File
+ 8
- 0
custom/conf/app.example.ini
View File
+ 4
- 0
docs/content/doc/advanced/config-cheat-sheet.en-us.md
View File
+ 5
- 1
docs/content/doc/advanced/config-cheat-sheet.zh-cn.md
View File
+ 51
- 81
modules/indexer/code/bleve.go
View File
+ 3
- 50
modules/indexer/code/bleve_test.go
View File
+ 385
- 0
modules/indexer/code/elastic_search.go
View File
| @ -0,0 +1,385 @@ | |||||
| // Copyright 2020 The Gitea Authors. All rights reserved. | |||||
| // Use of this source code is governed by a MIT-style | |||||
| // license that can be found in the LICENSE file. | |||||
| package code | |||||
| import ( | |||||
| "context" | |||||
| "encoding/json" | |||||
| "fmt" | |||||
| "strconv" | |||||
| "strings" | |||||
| "time" | |||||
| "code.gitea.io/gitea/models" | |||||
| "code.gitea.io/gitea/modules/analyze" | |||||
| "code.gitea.io/gitea/modules/base" | |||||
| "code.gitea.io/gitea/modules/charset" | |||||
| "code.gitea.io/gitea/modules/git" | |||||
| "code.gitea.io/gitea/modules/log" | |||||
| "code.gitea.io/gitea/modules/setting" | |||||
| "code.gitea.io/gitea/modules/timeutil" | |||||
| "github.com/go-enry/go-enry/v2" | |||||
| "github.com/olivere/elastic/v7" | |||||
| ) | |||||
| const ( | |||||
| esRepoIndexerLatestVersion = 1 | |||||
| ) | |||||
| var ( | |||||
| _ Indexer = &ElasticSearchIndexer{} | |||||
| ) | |||||
| // ElasticSearchIndexer implements Indexer interface | |||||
| type ElasticSearchIndexer struct { | |||||
| client *elastic.Client | |||||
| indexerAliasName string | |||||
| } | |||||
| type elasticLogger struct { | |||||
| *log.Logger | |||||
| } | |||||
| func (l elasticLogger) Printf(format string, args ...interface{}) { | |||||
| _ = l.Logger.Log(2, l.Logger.GetLevel(), format, args...) | |||||
| } | |||||
| // NewElasticSearchIndexer creates a new elasticsearch indexer | |||||
| func NewElasticSearchIndexer(url, indexerName string) (*ElasticSearchIndexer, bool, error) { | |||||
| opts := []elastic.ClientOptionFunc{ | |||||
| elastic.SetURL(url), | |||||
| elastic.SetSniff(false), | |||||
| elastic.SetHealthcheckInterval(10 * time.Second), | |||||
| elastic.SetGzip(false), | |||||
| } | |||||
| logger := elasticLogger{log.GetLogger(log.DEFAULT)} | |||||
| if logger.GetLevel() == log.TRACE || logger.GetLevel() == log.DEBUG { | |||||
| opts = append(opts, elastic.SetTraceLog(logger)) | |||||
| } else if logger.GetLevel() == log.ERROR || logger.GetLevel() == log.CRITICAL || logger.GetLevel() == log.FATAL { | |||||
| opts = append(opts, elastic.SetErrorLog(logger)) | |||||
| } else if logger.GetLevel() == log.INFO || logger.GetLevel() == log.WARN { | |||||
| opts = append(opts, elastic.SetInfoLog(logger)) | |||||
| } | |||||
| client, err := elastic.NewClient(opts...) | |||||
| if err != nil { | |||||
| return nil, false, err | |||||
| } | |||||
| indexer := &ElasticSearchIndexer{ | |||||
| client: client, | |||||
| indexerAliasName: indexerName, | |||||
| } | |||||
| exists, err := indexer.init() | |||||
| return indexer, !exists, err | |||||
| } | |||||
| const ( | |||||
| defaultMapping = `{ | |||||
| "mappings": { | |||||
| "properties": { | |||||
| "repo_id": { | |||||
| "type": "long", | |||||
| "index": true | |||||
| }, | |||||
| "content": { | |||||
| "type": "text", | |||||
| "index": true | |||||
| }, | |||||
| "commit_id": { | |||||
| "type": "keyword", | |||||
| "index": true | |||||
| }, | |||||
| "language": { | |||||
| "type": "keyword", | |||||
| "index": true | |||||
| }, | |||||
| "updated_at": { | |||||
| "type": "long", | |||||
| "index": true | |||||
| } | |||||
| } | |||||
| } | |||||
| }` | |||||
| ) | |||||
| func (b *ElasticSearchIndexer) realIndexerName() string { | |||||
| return fmt.Sprintf("%s.v%d", b.indexerAliasName, esRepoIndexerLatestVersion) | |||||
| } | |||||
| // Init will initialize the indexer | |||||
| func (b *ElasticSearchIndexer) init() (bool, error) { | |||||
| ctx := context.Background() | |||||
| exists, err := b.client.IndexExists(b.realIndexerName()).Do(ctx) | |||||
| if err != nil { | |||||
| return false, err | |||||
| } | |||||
| if !exists { | |||||
| var mapping = defaultMapping | |||||
| createIndex, err := b.client.CreateIndex(b.realIndexerName()).BodyString(mapping).Do(ctx) | |||||
| if err != nil { | |||||
| return false, err | |||||
| } | |||||
| if !createIndex.Acknowledged { | |||||
| return false, fmt.Errorf("create index %s with %s failed", b.realIndexerName(), mapping) | |||||
| } | |||||
| } | |||||
| // check version | |||||
| r, err := b.client.Aliases().Do(ctx) | |||||
| if err != nil { | |||||
| return false, err | |||||
| } | |||||
| realIndexerNames := r.IndicesByAlias(b.indexerAliasName) | |||||
| if len(realIndexerNames) < 1 { | |||||
| res, err := b.client.Alias(). | |||||
| Add(b.realIndexerName(), b.indexerAliasName). | |||||
| Do(ctx) | |||||
| if err != nil { | |||||
| return false, err | |||||
| } | |||||
| if !res.Acknowledged { | |||||
| return false, fmt.Errorf("") | |||||
| } | |||||
| } else if len(realIndexerNames) >= 1 && realIndexerNames[0] < b.realIndexerName() { | |||||
| log.Warn("Found older gitea indexer named %s, but we will create a new one %s and keep the old NOT DELETED. You can delete the old version after the upgrade succeed.", | |||||
| realIndexerNames[0], b.realIndexerName()) | |||||
| res, err := b.client.Alias(). | |||||
| Remove(realIndexerNames[0], b.indexerAliasName). | |||||
| Add(b.realIndexerName(), b.indexerAliasName). | |||||
| Do(ctx) | |||||
| if err != nil { | |||||
| return false, err | |||||
| } | |||||
| if !res.Acknowledged { | |||||
| return false, fmt.Errorf("") | |||||
| } | |||||
| } | |||||
| return exists, nil | |||||
| } | |||||
| func (b *ElasticSearchIndexer) addUpdate(sha string, update fileUpdate, repo *models.Repository) ([]elastic.BulkableRequest, error) { | |||||
| stdout, err := git.NewCommand("cat-file", "-s", update.BlobSha). | |||||
| RunInDir(repo.RepoPath()) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } | |||||
| if size, err := strconv.Atoi(strings.TrimSpace(stdout)); err != nil { | |||||
| return nil, fmt.Errorf("Misformatted git cat-file output: %v", err) | |||||
| } else if int64(size) > setting.Indexer.MaxIndexerFileSize { | |||||
| return []elastic.BulkableRequest{b.addDelete(update.Filename, repo)}, nil | |||||
| } | |||||
| fileContents, err := git.NewCommand("cat-file", "blob", update.BlobSha). | |||||
| RunInDirBytes(repo.RepoPath()) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } else if !base.IsTextFile(fileContents) { | |||||
| // FIXME: UTF-16 files will probably fail here | |||||
| return nil, nil | |||||
| } | |||||
| id := filenameIndexerID(repo.ID, update.Filename) | |||||
| return []elastic.BulkableRequest{ | |||||
| elastic.NewBulkIndexRequest(). | |||||
| Index(b.indexerAliasName). | |||||
| Id(id). | |||||
| Doc(map[string]interface{}{ | |||||
| "repo_id": repo.ID, | |||||
| "content": string(charset.ToUTF8DropErrors(fileContents)), | |||||
| "commit_id": sha, | |||||
| "language": analyze.GetCodeLanguage(update.Filename, fileContents), | |||||
| "updated_at": timeutil.TimeStampNow(), | |||||
| }), | |||||
| }, nil | |||||
| } | |||||
| func (b *ElasticSearchIndexer) addDelete(filename string, repo *models.Repository) elastic.BulkableRequest { | |||||
| id := filenameIndexerID(repo.ID, filename) | |||||
| return elastic.NewBulkDeleteRequest(). | |||||
| Index(b.indexerAliasName). | |||||
| Id(id) | |||||
| } | |||||
| // Index will save the index data | |||||
| func (b *ElasticSearchIndexer) Index(repo *models.Repository, sha string, changes *repoChanges) error { | |||||
| reqs := make([]elastic.BulkableRequest, 0) | |||||
| for _, update := range changes.Updates { | |||||
| updateReqs, err := b.addUpdate(sha, update, repo) | |||||
| if err != nil { | |||||
| return err | |||||
| } | |||||
| if len(updateReqs) > 0 { | |||||
| reqs = append(reqs, updateReqs...) | |||||
| } | |||||
| } | |||||
| for _, filename := range changes.RemovedFilenames { | |||||
| reqs = append(reqs, b.addDelete(filename, repo)) | |||||
| } | |||||
| if len(reqs) > 0 { | |||||
| _, err := b.client.Bulk(). | |||||
| Index(b.indexerAliasName). | |||||
| Add(reqs...). | |||||
| Do(context.Background()) | |||||
| return err | |||||
| } | |||||
| return nil | |||||
| } | |||||
| // Delete deletes indexes by ids | |||||
| func (b *ElasticSearchIndexer) Delete(repoID int64) error { | |||||
| _, err := b.client.DeleteByQuery(b.indexerAliasName). | |||||
| Query(elastic.NewTermsQuery("repo_id", repoID)). | |||||
| Do(context.Background()) | |||||
| return err | |||||
| } | |||||
| func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) { | |||||
| hits := make([]*SearchResult, 0, pageSize) | |||||
| for _, hit := range searchResult.Hits.Hits { | |||||
| // FIXME: There is no way to get the position the keyword on the content currently on the same request. | |||||
| // So we get it from content, this may made the query slower. See | |||||
| // https://discuss.elastic.co/t/fetching-position-of-keyword-in-matched-document/94291 | |||||
| var startIndex, endIndex int = -1, -1 | |||||
| c, ok := hit.Highlight["content"] | |||||
| if ok && len(c) > 0 { | |||||
| var subStr = make([]rune, 0, len(kw)) | |||||
| startIndex = strings.IndexFunc(c[0], func(r rune) bool { | |||||
| if len(subStr) >= len(kw) { | |||||
| subStr = subStr[1:] | |||||
| } | |||||
| subStr = append(subStr, r) | |||||
| return strings.EqualFold(kw, string(subStr)) | |||||
| }) | |||||
| if startIndex > -1 { | |||||
| endIndex = startIndex + len(kw) | |||||
| } else { | |||||
| panic(fmt.Sprintf("1===%#v", hit.Highlight)) | |||||
| } | |||||
| } else { | |||||
| panic(fmt.Sprintf("2===%#v", hit.Highlight)) | |||||
| } | |||||
| repoID, fileName := parseIndexerID(hit.Id) | |||||
| var res = make(map[string]interface{}) | |||||
| if err := json.Unmarshal(hit.Source, &res); err != nil { | |||||
| return 0, nil, nil, err | |||||
| } | |||||
| language := res["language"].(string) | |||||
| hits = append(hits, &SearchResult{ | |||||
| RepoID: repoID, | |||||
| Filename: fileName, | |||||
| CommitID: res["commit_id"].(string), | |||||
| Content: res["content"].(string), | |||||
| UpdatedUnix: timeutil.TimeStamp(res["updated_at"].(float64)), | |||||
| Language: language, | |||||
| StartIndex: startIndex, | |||||
| EndIndex: endIndex, | |||||
| Color: enry.GetColor(language), | |||||
| }) | |||||
| } | |||||
| return searchResult.TotalHits(), hits, extractAggs(searchResult), nil | |||||
| } | |||||
| func extractAggs(searchResult *elastic.SearchResult) []*SearchResultLanguages { | |||||
| var searchResultLanguages []*SearchResultLanguages | |||||
| agg, found := searchResult.Aggregations.Terms("language") | |||||
| if found { | |||||
| searchResultLanguages = make([]*SearchResultLanguages, 0, 10) | |||||
| for _, bucket := range agg.Buckets { | |||||
| searchResultLanguages = append(searchResultLanguages, &SearchResultLanguages{ | |||||
| Language: bucket.Key.(string), | |||||
| Color: enry.GetColor(bucket.Key.(string)), | |||||
| Count: int(bucket.DocCount), | |||||
| }) | |||||
| } | |||||
| } | |||||
| return searchResultLanguages | |||||
| } | |||||
| // Search searches for codes and language stats by given conditions. | |||||
| func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) { | |||||
| kwQuery := elastic.NewMultiMatchQuery(keyword, "content") | |||||
| query := elastic.NewBoolQuery() | |||||
| query = query.Must(kwQuery) | |||||
| if len(repoIDs) > 0 { | |||||
| var repoStrs = make([]interface{}, 0, len(repoIDs)) | |||||
| for _, repoID := range repoIDs { | |||||
| repoStrs = append(repoStrs, repoID) | |||||
| } | |||||
| repoQuery := elastic.NewTermsQuery("repo_id", repoStrs...) | |||||
| query = query.Must(repoQuery) | |||||
| } | |||||
| var ( | |||||
| start int | |||||
| kw = "<em>" + keyword + "</em>" | |||||
| aggregation = elastic.NewTermsAggregation().Field("language").Size(10).OrderByCountDesc() | |||||
| ) | |||||
| if page > 0 { | |||||
| start = (page - 1) * pageSize | |||||
| } | |||||
| if len(language) == 0 { | |||||
| searchResult, err := b.client.Search(). | |||||
| Index(b.indexerAliasName). | |||||
| Aggregation("language", aggregation). | |||||
| Query(query). | |||||
| Highlight(elastic.NewHighlight().Field("content")). | |||||
| Sort("repo_id", true). | |||||
| From(start).Size(pageSize). | |||||
| Do(context.Background()) | |||||
| if err != nil { | |||||
| return 0, nil, nil, err | |||||
| } | |||||
| return convertResult(searchResult, kw, pageSize) | |||||
| } | |||||
| langQuery := elastic.NewMatchQuery("language", language) | |||||
| countResult, err := b.client.Search(). | |||||
| Index(b.indexerAliasName). | |||||
| Aggregation("language", aggregation). | |||||
| Query(query). | |||||
| Size(0). // We only needs stats information | |||||
| Do(context.Background()) | |||||
| if err != nil { | |||||
| return 0, nil, nil, err | |||||
| } | |||||
| query = query.Must(langQuery) | |||||
| searchResult, err := b.client.Search(). | |||||
| Index(b.indexerAliasName). | |||||
| Query(query). | |||||
| Highlight(elastic.NewHighlight().Field("content")). | |||||
| Sort("repo_id", true). | |||||
| From(start).Size(pageSize). | |||||
| Do(context.Background()) | |||||
| if err != nil { | |||||
| return 0, nil, nil, err | |||||
| } | |||||
| total, hits, _, err := convertResult(searchResult, kw, pageSize) | |||||
| return total, hits, extractAggs(countResult), err | |||||
| } | |||||
| // Close implements indexer | |||||
| func (b *ElasticSearchIndexer) Close() {} | |||||
+ 36
- 0
modules/indexer/code/elastic_search_test.go
View File
| @ -0,0 +1,36 @@ | |||||
| // Copyright 2020 The Gitea Authors. All rights reserved. | |||||
| // Use of this source code is governed by a MIT-style | |||||
| // license that can be found in the LICENSE file. | |||||
| package code | |||||
| import ( | |||||
| "os" | |||||
| "testing" | |||||
| "code.gitea.io/gitea/models" | |||||
| "github.com/stretchr/testify/assert" | |||||
| ) | |||||
| func TestESIndexAndSearch(t *testing.T) { | |||||
| models.PrepareTestEnv(t) | |||||
| u := os.Getenv("TEST_INDEXER_CODE_ES_URL") | |||||
| if u == "" { | |||||
| t.SkipNow() | |||||
| return | |||||
| } | |||||
| indexer, _, err := NewElasticSearchIndexer(u, "gitea_codes") | |||||
| if err != nil { | |||||
| assert.Fail(t, "Unable to create ES indexer Error: %v", err) | |||||
| if indexer != nil { | |||||
| indexer.Close() | |||||
| } | |||||
| return | |||||
| } | |||||
| defer indexer.Close() | |||||
| testIndexer("elastic_search", t, indexer) | |||||
| } | |||||
+ 71
- 19
modules/indexer/code/indexer.go
View File
+ 83
- 0
modules/indexer/code/indexer_test.go
View File
| @ -0,0 +1,83 @@ | |||||
| // Copyright 2020 The Gitea Authors. All rights reserved. | |||||
| // Use of this source code is governed by a MIT-style | |||||
| // license that can be found in the LICENSE file. | |||||
| package code | |||||
| import ( | |||||
| "path/filepath" | |||||
| "testing" | |||||
| "code.gitea.io/gitea/models" | |||||
| "github.com/stretchr/testify/assert" | |||||
| ) | |||||
| func TestMain(m *testing.M) { | |||||
| models.MainTest(m, filepath.Join("..", "..", "..")) | |||||
| } | |||||
| func testIndexer(name string, t *testing.T, indexer Indexer) { | |||||
| t.Run(name, func(t *testing.T) { | |||||
| var repoID int64 = 1 | |||||
| err := index(indexer, repoID) | |||||
| assert.NoError(t, err) | |||||
| var ( | |||||
| keywords = []struct { | |||||
| RepoIDs []int64 | |||||
| Keyword string | |||||
| IDs []int64 | |||||
| Langs int | |||||
| }{ | |||||
| { | |||||
| RepoIDs: nil, | |||||
| Keyword: "Description", | |||||
| IDs: []int64{repoID}, | |||||
| Langs: 1, | |||||
| }, | |||||
| { | |||||
| RepoIDs: []int64{2}, | |||||
| Keyword: "Description", | |||||
| IDs: []int64{}, | |||||
| Langs: 0, | |||||
| }, | |||||
| { | |||||
| RepoIDs: nil, | |||||
| Keyword: "repo1", | |||||
| IDs: []int64{repoID}, | |||||
| Langs: 1, | |||||
| }, | |||||
| { | |||||
| RepoIDs: []int64{2}, | |||||
| Keyword: "repo1", | |||||
| IDs: []int64{}, | |||||
| Langs: 0, | |||||
| }, | |||||
| { | |||||
| RepoIDs: nil, | |||||
| Keyword: "non-exist", | |||||
| IDs: []int64{}, | |||||
| Langs: 0, | |||||
| }, | |||||
| } | |||||
| ) | |||||
| for _, kw := range keywords { | |||||
| t.Run(kw.Keyword, func(t *testing.T) { | |||||
| total, res, langs, err := indexer.Search(kw.RepoIDs, "", kw.Keyword, 1, 10) | |||||
| assert.NoError(t, err) | |||||
| assert.EqualValues(t, len(kw.IDs), total) | |||||
| assert.EqualValues(t, kw.Langs, len(langs)) | |||||
| var ids = make([]int64, 0, len(res)) | |||||
| for _, hit := range res { | |||||
| ids = append(ids, hit.RepoID) | |||||
| assert.EqualValues(t, "# repo1\n\nDescription for repo1", hit.Content) | |||||
| } | |||||
| assert.EqualValues(t, kw.IDs, ids) | |||||
| }) | |||||
| } | |||||
| assert.NoError(t, indexer.Delete(repoID)) | |||||
| }) | |||||
| } | |||||