Browse Source

Global code search support (#3664)

* add global code search on explore

* fix bug when no anyone public repos

* change the icon

* fix typo and add UnitTypeCode check for login non-admin user

* fix ui description when no match
for-closed-social
Lunny Xiao 6 years ago
committed by GitHub
parent
commit
9e5d0a09eb
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 238 additions and 9 deletions
  1. +6
    -0
      models/repo.go
  2. +25
    -0
      models/repo_list.go
  3. +21
    -6
      modules/indexer/repo.go
  4. +4
    -2
      modules/search/search.go
  5. +3
    -0
      options/locale/locale_en-US.ini
  6. +116
    -0
      routers/home.go
  7. +2
    -1
      routers/repo/search.go
  8. +1
    -0
      routers/routes/routes.go
  9. +55
    -0
      templates/explore/code.tmpl
  10. +5
    -0
      templates/explore/navbar.tmpl

+ 6
- 0
models/repo.go View File

@ -1945,6 +1945,12 @@ func GetRepositoryByID(id int64) (*Repository, error) {
return getRepositoryByID(x, id)
}
// GetRepositoriesMapByIDs returns the repositories by given id slice.
func GetRepositoriesMapByIDs(ids []int64) (map[int64]*Repository, error) {
var repos = make(map[int64]*Repository, len(ids))
return repos, x.In("id", ids).Find(&repos)
}
// GetUserRepositories returns a list of repositories of given user.
func GetUserRepositories(userID int64, private bool, page, pageSize int, orderBy string) ([]*Repository, error) {
if len(orderBy) == 0 {

+ 25
- 0
models/repo_list.go View File

@ -249,3 +249,28 @@ func SearchRepositoryByName(opts *SearchRepoOptions) (RepositoryList, int64, err
return repos, count, nil
}
// FindUserAccessibleRepoIDs find all accessible repositories' ID by user's id
func FindUserAccessibleRepoIDs(userID int64) ([]int64, error) {
var accessCond builder.Cond = builder.Eq{"is_private": false}
if userID > 0 {
accessCond = accessCond.Or(
builder.Eq{"owner_id": userID},
builder.And(
builder.Expr("id IN (SELECT repo_id FROM `access` WHERE access.user_id = ?)", userID),
builder.Neq{"owner_id": userID},
),
)
}
repoIDs := make([]int64, 0, 10)
if err := x.
Table("repository").
Cols("id").
Where(accessCond).
Find(&repoIDs); err != nil {
return nil, fmt.Errorf("FindUserAccesibleRepoIDs: %v", err)
}
return repoIDs, nil
}

+ 21
- 6
modules/indexer/repo.go View File

@ -16,6 +16,7 @@ import (
"github.com/blevesearch/bleve/analysis/token/lowercase"
"github.com/blevesearch/bleve/analysis/token/unique"
"github.com/blevesearch/bleve/analysis/tokenizer/unicode"
"github.com/blevesearch/bleve/search/query"
"github.com/ethantkoenig/rupture"
)
@ -158,6 +159,7 @@ func DeleteRepoFromIndexer(repoID int64) error {
// RepoSearchResult result of performing a search in a repo
type RepoSearchResult struct {
RepoID int64
StartIndex int
EndIndex int
Filename string
@ -166,17 +168,29 @@ type RepoSearchResult struct {
// SearchRepoByKeyword searches for files in the specified repo.
// Returns the matching file-paths
func SearchRepoByKeyword(repoID int64, keyword string, page, pageSize int) (int64, []*RepoSearchResult, error) {
func SearchRepoByKeyword(repoIDs []int64, keyword string, page, pageSize int) (int64, []*RepoSearchResult, error) {
phraseQuery := bleve.NewMatchPhraseQuery(keyword)
phraseQuery.FieldVal = "Content"
phraseQuery.Analyzer = repoIndexerAnalyzer
indexerQuery := bleve.NewConjunctionQuery(
numericEqualityQuery(repoID, "RepoID"),
phraseQuery,
)
var indexerQuery query.Query
if len(repoIDs) > 0 {
var repoQueries = make([]query.Query, 0, len(repoIDs))
for _, repoID := range repoIDs {
repoQueries = append(repoQueries, numericEqualityQuery(repoID, "RepoID"))
}
indexerQuery = bleve.NewConjunctionQuery(
bleve.NewDisjunctionQuery(repoQueries...),
phraseQuery,
)
} else {
indexerQuery = phraseQuery
}
from := (page - 1) * pageSize
searchRequest := bleve.NewSearchRequestOptions(indexerQuery, pageSize, from, false)
searchRequest.Fields = []string{"Content"}
searchRequest.Fields = []string{"Content", "RepoID"}
searchRequest.IncludeLocations = true
result, err := repoIndexer.Search(searchRequest)
@ -199,6 +213,7 @@ func SearchRepoByKeyword(repoID int64, keyword string, page, pageSize int) (int6
}
}
searchResults[i] = &RepoSearchResult{
RepoID: int64(hit.Fields["RepoID"].(float64)),
StartIndex: startIndex,
EndIndex: endIndex,
Filename: filenameOfIndexerID(hit.ID),

+ 4
- 2
modules/search/search.go View File

@ -17,6 +17,7 @@ import (
// Result a search result to display
type Result struct {
RepoID int64
Filename string
HighlightClass string
LineNumbers []int
@ -98,6 +99,7 @@ func searchResult(result *indexer.RepoSearchResult, startIndex, endIndex int) (*
index += len(line)
}
return &Result{
RepoID: result.RepoID,
Filename: result.Filename,
HighlightClass: highlight.FileNameToHighlightClass(result.Filename),
LineNumbers: lineNumbers,
@ -106,12 +108,12 @@ func searchResult(result *indexer.RepoSearchResult, startIndex, endIndex int) (*
}
// PerformSearch perform a search on a repository
func PerformSearch(repoID int64, keyword string, page, pageSize int) (int, []*Result, error) {
func PerformSearch(repoIDs []int64, keyword string, page, pageSize int) (int, []*Result, error) {
if len(keyword) == 0 {
return 0, nil, nil
}
total, results, err := indexer.SearchRepoByKeyword(repoID, keyword, page, pageSize)
total, results, err := indexer.SearchRepoByKeyword(repoIDs, keyword, page, pageSize)
if err != nil {
return 0, nil, err
}

+ 3
- 0
options/locale/locale_en-US.ini View File

@ -169,9 +169,12 @@ repos = Repositories
users = Users
organizations = Organizations
search = Search
code = Code
repo_no_results = No matching repositories have been found.
user_no_results = No matching users have been found.
org_no_results = No matching organizations have been found.
code_no_results = No matching codes have been found.
code_search_results = Search results for "%s"
[auth]
create_new_account = Create Account

+ 116
- 0
routers/home.go View File

@ -11,6 +11,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/search"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/routers/user"
@ -27,6 +28,8 @@ const (
tplExploreUsers base.TplName = "explore/users"
// tplExploreOrganizations explore organizations page template
tplExploreOrganizations base.TplName = "explore/organizations"
// tplExploreCode explore code page template
tplExploreCode base.TplName = "explore/code"
)
// Home render home page
@ -49,6 +52,7 @@ func Home(ctx *context.Context) {
}
ctx.Data["PageIsHome"] = true
ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
ctx.HTML(200, tplHome)
}
@ -124,6 +128,7 @@ func RenderRepoSearch(ctx *context.Context, opts *RepoSearchOptions) {
ctx.Data["Total"] = count
ctx.Data["Page"] = paginater.New(int(count), opts.PageSize, page, 5)
ctx.Data["Repos"] = repos
ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
ctx.HTML(200, opts.TplName)
}
@ -133,6 +138,7 @@ func ExploreRepos(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("explore")
ctx.Data["PageIsExplore"] = true
ctx.Data["PageIsExploreRepositories"] = true
ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
var ownerID int64
if ctx.User != nil && !ctx.User.IsAdmin {
@ -194,6 +200,7 @@ func RenderUserSearch(ctx *context.Context, opts *models.SearchUserOptions, tplN
ctx.Data["Page"] = paginater.New(int(count), opts.PageSize, opts.Page, 5)
ctx.Data["Users"] = users
ctx.Data["ShowUserEmail"] = setting.UI.ShowUserEmail
ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
ctx.HTML(200, tplName)
}
@ -203,6 +210,7 @@ func ExploreUsers(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("explore")
ctx.Data["PageIsExplore"] = true
ctx.Data["PageIsExploreUsers"] = true
ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
RenderUserSearch(ctx, &models.SearchUserOptions{
Type: models.UserTypeIndividual,
@ -216,6 +224,7 @@ func ExploreOrganizations(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("explore")
ctx.Data["PageIsExplore"] = true
ctx.Data["PageIsExploreOrganizations"] = true
ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
RenderUserSearch(ctx, &models.SearchUserOptions{
Type: models.UserTypeOrganization,
@ -223,6 +232,113 @@ func ExploreOrganizations(ctx *context.Context) {
}, tplExploreOrganizations)
}
// ExploreCode render explore code page
func ExploreCode(ctx *context.Context) {
if !setting.Indexer.RepoIndexerEnabled {
ctx.Redirect("/explore", 302)
return
}
ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
ctx.Data["Title"] = ctx.Tr("explore")
ctx.Data["PageIsExplore"] = true
ctx.Data["PageIsExploreCode"] = true
keyword := strings.TrimSpace(ctx.Query("q"))
page := ctx.QueryInt("page")
if page <= 0 {
page = 1
}
var (
repoIDs []int64
err error
isAdmin bool
userID int64
)
if ctx.User != nil {
userID = ctx.User.ID
isAdmin = ctx.User.IsAdmin
}
// guest user or non-admin user
if ctx.User == nil || !isAdmin {
repoIDs, err = models.FindUserAccessibleRepoIDs(userID)
if err != nil {
ctx.ServerError("SearchResults", err)
return
}
}
var (
total int
searchResults []*search.Result
)
// if non-admin login user, we need check UnitTypeCode at first
if ctx.User != nil && len(repoIDs) > 0 {
repoMaps, err := models.GetRepositoriesMapByIDs(repoIDs)
if err != nil {
ctx.ServerError("SearchResults", err)
return
}
var rightRepoMap = make(map[int64]*models.Repository, len(repoMaps))
repoIDs = make([]int64, 0, len(repoMaps))
for id, repo := range repoMaps {
if repo.CheckUnitUser(userID, isAdmin, models.UnitTypeCode) {
rightRepoMap[id] = repo
repoIDs = append(repoIDs, id)
}
}
ctx.Data["RepoMaps"] = rightRepoMap
total, searchResults, err = search.PerformSearch(repoIDs, keyword, page, setting.UI.RepoSearchPagingNum)
if err != nil {
ctx.ServerError("SearchResults", err)
return
}
// if non-login user or isAdmin, no need to check UnitTypeCode
} else if (ctx.User == nil && len(repoIDs) > 0) || isAdmin {
total, searchResults, err = search.PerformSearch(repoIDs, keyword, page, setting.UI.RepoSearchPagingNum)
if err != nil {
ctx.ServerError("SearchResults", err)
return
}
var loadRepoIDs = make([]int64, 0, len(searchResults))
for _, result := range searchResults {
var find bool
for _, id := range loadRepoIDs {
if id == result.RepoID {
find = true
break
}
}
if !find {
loadRepoIDs = append(loadRepoIDs, result.RepoID)
}
}
repoMaps, err := models.GetRepositoriesMapByIDs(loadRepoIDs)
if err != nil {
ctx.ServerError("SearchResults", err)
return
}
ctx.Data["RepoMaps"] = repoMaps
}
ctx.Data["Keyword"] = keyword
pager := paginater.New(total, setting.UI.RepoSearchPagingNum, page, 5)
ctx.Data["Page"] = pager
ctx.Data["SearchResults"] = searchResults
ctx.Data["RequireHighlightJS"] = true
ctx.Data["PageIsViewCode"] = true
ctx.HTML(200, tplExploreCode)
}
// NotFound render 404 page
func NotFound(ctx *context.Context) {
ctx.Data["Title"] = "Page Not Found"

+ 2
- 1
routers/repo/search.go View File

@ -29,7 +29,8 @@ func Search(ctx *context.Context) {
if page <= 0 {
page = 1
}
total, searchResults, err := search.PerformSearch(ctx.Repo.Repository.ID, keyword, page, setting.UI.RepoSearchPagingNum)
total, searchResults, err := search.PerformSearch([]int64{ctx.Repo.Repository.ID},
keyword, page, setting.UI.RepoSearchPagingNum)
if err != nil {
ctx.ServerError("SearchResults", err)
return

+ 1
- 0
routers/routes/routes.go View File

@ -170,6 +170,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/repos", routers.ExploreRepos)
m.Get("/users", routers.ExploreUsers)
m.Get("/organizations", routers.ExploreOrganizations)
m.Get("/code", routers.ExploreCode)
}, ignSignIn)
m.Combo("/install", routers.InstallInit).Get(routers.Install).
Post(bindIgnErr(auth.InstallForm{}), routers.InstallPost)

+ 55
- 0
templates/explore/code.tmpl View File

@ -0,0 +1,55 @@
{{template "base/head" .}}
<div class="explore users">
{{template "explore/navbar" .}}
<div class="ui container">
<form class="ui form" style="max-width: 100%">
<div class="ui fluid action input">
<input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "explore.search"}}..." autofocus>
<input type="hidden" name="tab" value="{{$.TabName}}">
<button class="ui blue button">{{.i18n.Tr "explore.search"}}</button>
</div>
</form>
<div class="ui divider"></div>
<div class="ui user list">
{{if .SearchResults}}
<h3>
{{.i18n.Tr "explore.code_search_results" (.Keyword|Escape) | Str2html }}
</h3>
<div class="repository search">
{{range $result := .SearchResults}}
{{$repo := (index $.RepoMaps .RepoID)}}
<div class="diff-file-box diff-box file-content non-diff-file-content repo-search-result">
<h4 class="ui top attached normal header">
<span class="file"><a rel="nofollow" href="{{EscapePound $repo.HTMLURL}}">{{$repo.FullName}}</a> - {{.Filename}}</span>
<a class="ui basic grey tiny button" rel="nofollow" href="{{EscapePound $repo.HTMLURL}}/src/branch/{{$repo.DefaultBranch}}/{{EscapePound .Filename}}">{{$.i18n.Tr "repo.diff.view_file"}}</a>
</h4>
<div class="ui attached table segment">
<div class="file-body file-code code-view">
<table>
<tbody>
<tr>
<td class="lines-num">
{{range .LineNumbers}}
<a href="{{EscapePound $repo.HTMLURL}}/src/branch/{{$repo.DefaultBranch}}/{{EscapePound $result.Filename}}#L{{.}}"><span>{{.}}</span></a>
{{end}}
</td>
<td class="lines-code"><pre><code class="{{.HighlightClass}}"><ol class="linenums">{{.FormattedLines}}</ol></code></pre></td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
{{end}}
</div>
{{else}}
<div>{{$.i18n.Tr "explore.code_no_results"}}</div>
{{end}}
</div>
{{template "base/paginate" .}}
</div>
</div>
{{template "base/footer" .}}

+ 5
- 0
templates/explore/navbar.tmpl View File

@ -8,4 +8,9 @@
<a class="{{if .PageIsExploreOrganizations}}active{{end}} item" href="{{AppSubUrl}}/explore/organizations">
<span class="octicon octicon-organization"></span> {{.i18n.Tr "explore.organizations"}}
</a>
{{if .IsRepoIndexerEnabled}}
<a class="{{if .PageIsExploreCode}}active{{end}} item" href="{{AppSubUrl}}/explore/code">
<span class="octicon octicon-code"></span> {{.i18n.Tr "explore.code"}}
</a>
{{end}}
</div>

Loading…
Cancel
Save