Browse Source
Move all push update operations to a queue (#10133)
* Fix test * Add no queue for test only * improve code * Auto watch whatever branch operation * Fix lint * Rename noqueue to immediate * Remove old PushUpdate function * Fix tests Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: techknowlogick <techknowlogick@gitea.io>for-closed-social
Lunny Xiao
4 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 552 additions and 585 deletions
Split View
Diff Options
-
+4 -1integrations/mssql.ini.tmpl
-
+4 -1integrations/mysql.ini.tmpl
-
+4 -0integrations/mysql8.ini.tmpl
-
+4 -1integrations/pgsql.ini.tmpl
-
+3 -0integrations/sqlite.ini.tmpl
-
+0 -141modules/repofiles/action.go
-
+0 -121modules/repofiles/action_test.go
-
+0 -298modules/repofiles/update.go
-
+3 -5routers/api/v1/repo/branch.go
-
+4 -0routers/init.go
-
+4 -4routers/private/hook.go
-
+5 -8routers/repo/branch.go
-
+2 -5routers/repo/pull.go
-
+371 -0services/repository/push.go
-
+139 -0services/repository/push_test.go
-
+5 -0services/repository/repository.go
+ 4
- 1
integrations/mssql.ini.tmpl
View File
+ 4
- 1
integrations/mysql.ini.tmpl
View File
+ 4
- 0
integrations/mysql8.ini.tmpl
View File
+ 4
- 1
integrations/pgsql.ini.tmpl
View File
+ 3
- 0
integrations/sqlite.ini.tmpl
View File
+ 0
- 141
modules/repofiles/action.go
View File
+ 0
- 121
modules/repofiles/action_test.go
View File
+ 0
- 298
modules/repofiles/update.go
View File
+ 3
- 5
routers/api/v1/repo/branch.go
View File
+ 4
- 0
routers/init.go
View File
+ 4
- 4
routers/private/hook.go
View File
+ 5
- 8
routers/repo/branch.go
View File
+ 2
- 5
routers/repo/pull.go
View File
+ 371
- 0
services/repository/push.go
View File
| @ -0,0 +1,371 @@ | |||
| // Copyright 2020 The Gitea Authors. All rights reserved. | |||
| // Use of this source code is governed by a MIT-style | |||
| // license that can be found in the LICENSE file. | |||
| package repository | |||
| import ( | |||
| "container/list" | |||
| "encoding/json" | |||
| "fmt" | |||
| "strings" | |||
| "code.gitea.io/gitea/models" | |||
| "code.gitea.io/gitea/modules/cache" | |||
| "code.gitea.io/gitea/modules/git" | |||
| "code.gitea.io/gitea/modules/graceful" | |||
| "code.gitea.io/gitea/modules/log" | |||
| "code.gitea.io/gitea/modules/notification" | |||
| "code.gitea.io/gitea/modules/queue" | |||
| "code.gitea.io/gitea/modules/repofiles" | |||
| repo_module "code.gitea.io/gitea/modules/repository" | |||
| "code.gitea.io/gitea/modules/setting" | |||
| pull_service "code.gitea.io/gitea/services/pull" | |||
| ) | |||
| // PushUpdateOptions defines the push update options | |||
| type PushUpdateOptions struct { | |||
| PusherID int64 | |||
| PusherName string | |||
| RepoUserName string | |||
| RepoName string | |||
| RefFullName string | |||
| OldCommitID string | |||
| NewCommitID string | |||
| } | |||
| // IsNewRef return true if it's a first-time push to a branch, tag or etc. | |||
| func (opts PushUpdateOptions) IsNewRef() bool { | |||
| return opts.OldCommitID == git.EmptySHA | |||
| } | |||
| // IsDelRef return true if it's a deletion to a branch or tag | |||
| func (opts PushUpdateOptions) IsDelRef() bool { | |||
| return opts.NewCommitID == git.EmptySHA | |||
| } | |||
| // IsUpdateRef return true if it's an update operation | |||
| func (opts PushUpdateOptions) IsUpdateRef() bool { | |||
| return !opts.IsNewRef() && !opts.IsDelRef() | |||
| } | |||
| // IsTag return true if it's an operation to a tag | |||
| func (opts PushUpdateOptions) IsTag() bool { | |||
| return strings.HasPrefix(opts.RefFullName, git.TagPrefix) | |||
| } | |||
| // IsNewTag return true if it's a creation to a tag | |||
| func (opts PushUpdateOptions) IsNewTag() bool { | |||
| return opts.IsTag() && opts.IsNewRef() | |||
| } | |||
| // IsDelTag return true if it's a deletion to a tag | |||
| func (opts PushUpdateOptions) IsDelTag() bool { | |||
| return opts.IsTag() && opts.IsDelRef() | |||
| } | |||
| // IsBranch return true if it's a push to branch | |||
| func (opts PushUpdateOptions) IsBranch() bool { | |||
| return strings.HasPrefix(opts.RefFullName, git.BranchPrefix) | |||
| } | |||
| // IsNewBranch return true if it's the first-time push to a branch | |||
| func (opts PushUpdateOptions) IsNewBranch() bool { | |||
| return opts.IsBranch() && opts.IsNewRef() | |||
| } | |||
| // IsUpdateBranch return true if it's not the first push to a branch | |||
| func (opts PushUpdateOptions) IsUpdateBranch() bool { | |||
| return opts.IsBranch() && opts.IsUpdateRef() | |||
| } | |||
| // IsDelBranch return true if it's a deletion to a branch | |||
| func (opts PushUpdateOptions) IsDelBranch() bool { | |||
| return opts.IsBranch() && opts.IsDelRef() | |||
| } | |||
| // TagName returns simple tag name if it's an operation to a tag | |||
| func (opts PushUpdateOptions) TagName() string { | |||
| return opts.RefFullName[len(git.TagPrefix):] | |||
| } | |||
| // BranchName returns simple branch name if it's an operation to branch | |||
| func (opts PushUpdateOptions) BranchName() string { | |||
| return opts.RefFullName[len(git.BranchPrefix):] | |||
| } | |||
| // RepoFullName returns repo full name | |||
| func (opts PushUpdateOptions) RepoFullName() string { | |||
| return opts.RepoUserName + "/" + opts.RepoName | |||
| } | |||
| // pushQueue represents a queue to handle update pull request tests | |||
| var pushQueue queue.Queue | |||
| // handle passed PR IDs and test the PRs | |||
| func handle(data ...queue.Data) { | |||
| for _, datum := range data { | |||
| opts := datum.([]*PushUpdateOptions) | |||
| if err := pushUpdates(opts); err != nil { | |||
| log.Error("pushUpdate failed: %v", err) | |||
| } | |||
| } | |||
| } | |||
| func initPushQueue() error { | |||
| pushQueue = queue.CreateQueue("push_update", handle, []*PushUpdateOptions{}).(queue.Queue) | |||
| if pushQueue == nil { | |||
| return fmt.Errorf("Unable to create push_update Queue") | |||
| } | |||
| go graceful.GetManager().RunWithShutdownFns(pushQueue.Run) | |||
| return nil | |||
| } | |||
| // PushUpdate is an alias of PushUpdates for single push update options | |||
| func PushUpdate(opts *PushUpdateOptions) error { | |||
| return PushUpdates([]*PushUpdateOptions{opts}) | |||
| } | |||
| // PushUpdates adds a push update to push queue | |||
| func PushUpdates(opts []*PushUpdateOptions) error { | |||
| if len(opts) == 0 { | |||
| return nil | |||
| } | |||
| for _, opt := range opts { | |||
| if opt.IsNewRef() && opt.IsDelRef() { | |||
| return fmt.Errorf("Old and new revisions are both %s", git.EmptySHA) | |||
| } | |||
| } | |||
| return pushQueue.Push(opts) | |||
| } | |||
| // pushUpdates generates push action history feeds for push updating multiple refs | |||
| func pushUpdates(optsList []*PushUpdateOptions) error { | |||
| if len(optsList) == 0 { | |||
| return nil | |||
| } | |||
| repo, err := models.GetRepositoryByOwnerAndName(optsList[0].RepoUserName, optsList[0].RepoName) | |||
| if err != nil { | |||
| return fmt.Errorf("GetRepositoryByOwnerAndName failed: %v", err) | |||
| } | |||
| repoPath := repo.RepoPath() | |||
| _, err = git.NewCommand("update-server-info").RunInDir(repoPath) | |||
| if err != nil { | |||
| return fmt.Errorf("Failed to call 'git update-server-info': %v", err) | |||
| } | |||
| gitRepo, err := git.OpenRepository(repoPath) | |||
| if err != nil { | |||
| return fmt.Errorf("OpenRepository: %v", err) | |||
| } | |||
| defer gitRepo.Close() | |||
| if err = repo.UpdateSize(models.DefaultDBContext()); err != nil { | |||
| log.Error("Failed to update size for repository: %v", err) | |||
| } | |||
| addTags := make([]string, 0, len(optsList)) | |||
| delTags := make([]string, 0, len(optsList)) | |||
| actions := make([]*commitRepoActionOptions, 0, len(optsList)) | |||
| var pusher *models.User | |||
| for _, opts := range optsList { | |||
| if opts.IsNewRef() && opts.IsDelRef() { | |||
| return fmt.Errorf("Old and new revisions are both %s", git.EmptySHA) | |||
| } | |||
| var commits = &repo_module.PushCommits{} | |||
| if opts.IsTag() { // If is tag reference { | |||
| tagName := opts.TagName() | |||
| if opts.IsDelRef() { | |||
| delTags = append(delTags, tagName) | |||
| } else { // is new tag | |||
| cache.Remove(repo.GetCommitsCountCacheKey(tagName, true)) | |||
| addTags = append(addTags, tagName) | |||
| } | |||
| } else if opts.IsBranch() { // If is branch reference | |||
| if pusher == nil || pusher.ID != opts.PusherID { | |||
| var err error | |||
| if pusher, err = models.GetUserByID(opts.PusherID); err != nil { | |||
| return err | |||
| } | |||
| } | |||
| branch := opts.BranchName() | |||
| if !opts.IsDelRef() { | |||
| // Clear cache for branch commit count | |||
| cache.Remove(repo.GetCommitsCountCacheKey(opts.BranchName(), true)) | |||
| newCommit, err := gitRepo.GetCommit(opts.NewCommitID) | |||
| if err != nil { | |||
| return fmt.Errorf("gitRepo.GetCommit: %v", err) | |||
| } | |||
| // Push new branch. | |||
| var l *list.List | |||
| if opts.IsNewRef() { | |||
| l, err = newCommit.CommitsBeforeLimit(10) | |||
| if err != nil { | |||
| return fmt.Errorf("newCommit.CommitsBeforeLimit: %v", err) | |||
| } | |||
| } else { | |||
| l, err = newCommit.CommitsBeforeUntil(opts.OldCommitID) | |||
| if err != nil { | |||
| return fmt.Errorf("newCommit.CommitsBeforeUntil: %v", err) | |||
| } | |||
| } | |||
| commits = repo_module.ListToPushCommits(l) | |||
| if err = models.RemoveDeletedBranch(repo.ID, branch); err != nil { | |||
| log.Error("models.RemoveDeletedBranch %s/%s failed: %v", repo.ID, branch, err) | |||
| } | |||
| log.Trace("TriggerTask '%s/%s' by %s", repo.Name, branch, pusher.Name) | |||
| go pull_service.AddTestPullRequestTask(pusher, repo.ID, branch, true, opts.OldCommitID, opts.NewCommitID) | |||
| } else if err = pull_service.CloseBranchPulls(pusher, repo.ID, branch); err != nil { | |||
| // close all related pulls | |||
| log.Error("close related pull request failed: %v", err) | |||
| } | |||
| // Even if user delete a branch on a repository which he didn't watch, he will be watch that. | |||
| if err = models.WatchIfAuto(opts.PusherID, repo.ID, true); err != nil { | |||
| log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err) | |||
| } | |||
| } | |||
| actions = append(actions, &commitRepoActionOptions{ | |||
| PushUpdateOptions: *opts, | |||
| Pusher: pusher, | |||
| RepoOwnerID: repo.OwnerID, | |||
| Commits: commits, | |||
| }) | |||
| } | |||
| if err := repo_module.PushUpdateAddDeleteTags(repo, gitRepo, addTags, delTags); err != nil { | |||
| return fmt.Errorf("PushUpdateAddDeleteTags: %v", err) | |||
| } | |||
| if err := commitRepoAction(repo, gitRepo, actions...); err != nil { | |||
| return fmt.Errorf("commitRepoAction: %v", err) | |||
| } | |||
| return nil | |||
| } | |||
| // commitRepoActionOptions represent options of a new commit action. | |||
| type commitRepoActionOptions struct { | |||
| PushUpdateOptions | |||
| Pusher *models.User | |||
| RepoOwnerID int64 | |||
| Commits *repo_module.PushCommits | |||
| } | |||
| // commitRepoAction adds new commit action to the repository, and prepare | |||
| // corresponding webhooks. | |||
| func commitRepoAction(repo *models.Repository, gitRepo *git.Repository, optsList ...*commitRepoActionOptions) error { | |||
| actions := make([]*models.Action, len(optsList)) | |||
| for i, opts := range optsList { | |||
| if opts.Pusher == nil || opts.Pusher.Name != opts.PusherName { | |||
| var err error | |||
| opts.Pusher, err = models.GetUserByName(opts.PusherName) | |||
| if err != nil { | |||
| return fmt.Errorf("GetUserByName [%s]: %v", opts.PusherName, err) | |||
| } | |||
| } | |||
| refName := git.RefEndName(opts.RefFullName) | |||
| // Change default branch and empty status only if pushed ref is non-empty branch. | |||
| if repo.IsEmpty && opts.IsBranch() && !opts.IsDelRef() { | |||
| repo.DefaultBranch = refName | |||
| repo.IsEmpty = false | |||
| if refName != "master" { | |||
| if err := gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil { | |||
| if !git.IsErrUnsupportedVersion(err) { | |||
| return err | |||
| } | |||
| } | |||
| } | |||
| } | |||
| opType := models.ActionCommitRepo | |||
| // Check it's tag push or branch. | |||
| if opts.IsTag() { | |||
| opType = models.ActionPushTag | |||
| if opts.IsDelRef() { | |||
| opType = models.ActionDeleteTag | |||
| } | |||
| opts.Commits = &repo_module.PushCommits{} | |||
| } else if opts.IsDelRef() { | |||
| opType = models.ActionDeleteBranch | |||
| opts.Commits = &repo_module.PushCommits{} | |||
| } else { | |||
| // if not the first commit, set the compare URL. | |||
| if !opts.IsNewRef() { | |||
| opts.Commits.CompareURL = repo.ComposeCompareURL(opts.OldCommitID, opts.NewCommitID) | |||
| } | |||
| if err := repofiles.UpdateIssuesCommit(opts.Pusher, repo, opts.Commits.Commits, refName); err != nil { | |||
| log.Error("updateIssuesCommit: %v", err) | |||
| } | |||
| } | |||
| if len(opts.Commits.Commits) > setting.UI.FeedMaxCommitNum { | |||
| opts.Commits.Commits = opts.Commits.Commits[:setting.UI.FeedMaxCommitNum] | |||
| } | |||
| data, err := json.Marshal(opts.Commits) | |||
| if err != nil { | |||
| return fmt.Errorf("Marshal: %v", err) | |||
| } | |||
| actions[i] = &models.Action{ | |||
| ActUserID: opts.Pusher.ID, | |||
| ActUser: opts.Pusher, | |||
| OpType: opType, | |||
| Content: string(data), | |||
| RepoID: repo.ID, | |||
| Repo: repo, | |||
| RefName: refName, | |||
| IsPrivate: repo.IsPrivate, | |||
| } | |||
| var isHookEventPush = true | |||
| switch opType { | |||
| case models.ActionCommitRepo: // Push | |||
| if opts.IsNewBranch() { | |||
| notification.NotifyCreateRef(opts.Pusher, repo, "branch", opts.RefFullName) | |||
| } | |||
| case models.ActionDeleteBranch: // Delete Branch | |||
| notification.NotifyDeleteRef(opts.Pusher, repo, "branch", opts.RefFullName) | |||
| case models.ActionPushTag: // Create | |||
| notification.NotifyCreateRef(opts.Pusher, repo, "tag", opts.RefFullName) | |||
| case models.ActionDeleteTag: // Delete Tag | |||
| notification.NotifyDeleteRef(opts.Pusher, repo, "tag", opts.RefFullName) | |||
| default: | |||
| isHookEventPush = false | |||
| } | |||
| if isHookEventPush { | |||
| notification.NotifyPushCommits(opts.Pusher, repo, opts.RefFullName, opts.OldCommitID, opts.NewCommitID, opts.Commits) | |||
| } | |||
| } | |||
| // Change repository empty status and update last updated time. | |||
| if err := models.UpdateRepository(repo, false); err != nil { | |||
| return fmt.Errorf("UpdateRepository: %v", err) | |||
| } | |||
| if err := models.NotifyWatchers(actions...); err != nil { | |||
| return fmt.Errorf("NotifyWatchers: %v", err) | |||
| } | |||
| return nil | |||
| } | |||
+ 139
- 0
services/repository/push_test.go
View File
| @ -0,0 +1,139 @@ | |||
| // Copyright 2020 The Gitea Authors. All rights reserved. | |||
| // Use of this source code is governed by a MIT-style | |||
| // license that can be found in the LICENSE file. | |||
| package repository | |||
| import ( | |||
| "testing" | |||
| "code.gitea.io/gitea/models" | |||
| "code.gitea.io/gitea/modules/git" | |||
| repo_module "code.gitea.io/gitea/modules/repository" | |||
| "github.com/stretchr/testify/assert" | |||
| ) | |||
| func testCorrectRepoAction(t *testing.T, repo *models.Repository, gitRepo *git.Repository, opts *commitRepoActionOptions, actionBean *models.Action) { | |||
| models.AssertNotExistsBean(t, actionBean) | |||
| assert.NoError(t, commitRepoAction(repo, gitRepo, opts)) | |||
| models.AssertExistsAndLoadBean(t, actionBean) | |||
| models.CheckConsistencyFor(t, &models.Action{}) | |||
| } | |||
| func TestCommitRepoAction(t *testing.T) { | |||
| samples := []struct { | |||
| userID int64 | |||
| repositoryID int64 | |||
| commitRepoActionOptions commitRepoActionOptions | |||
| action models.Action | |||
| }{ | |||
| { | |||
| userID: 2, | |||
| repositoryID: 16, | |||
| commitRepoActionOptions: commitRepoActionOptions{ | |||
| PushUpdateOptions: PushUpdateOptions{ | |||
| RefFullName: "refName", | |||
| OldCommitID: "oldCommitID", | |||
| NewCommitID: "newCommitID", | |||
| }, | |||
| Commits: &repo_module.PushCommits{ | |||
| Commits: []*repo_module.PushCommit{ | |||
| { | |||
| Sha1: "69554a6", | |||
| CommitterEmail: "user2@example.com", | |||
| CommitterName: "User2", | |||
| AuthorEmail: "user2@example.com", | |||
| AuthorName: "User2", | |||
| Message: "not signed commit", | |||
| }, | |||
| { | |||
| Sha1: "27566bd", | |||
| CommitterEmail: "user2@example.com", | |||
| CommitterName: "User2", | |||
| AuthorEmail: "user2@example.com", | |||
| AuthorName: "User2", | |||
| Message: "good signed commit (with not yet validated email)", | |||
| }, | |||
| }, | |||
| Len: 2, | |||
| }, | |||
| }, | |||
| action: models.Action{ | |||
| OpType: models.ActionCommitRepo, | |||
| RefName: "refName", | |||
| }, | |||
| }, | |||
| { | |||
| userID: 2, | |||
| repositoryID: 1, | |||
| commitRepoActionOptions: commitRepoActionOptions{ | |||
| PushUpdateOptions: PushUpdateOptions{ | |||
| RefFullName: git.TagPrefix + "v1.1", | |||
| OldCommitID: git.EmptySHA, | |||
| NewCommitID: "newCommitID", | |||
| }, | |||
| Commits: &repo_module.PushCommits{}, | |||
| }, | |||
| action: models.Action{ | |||
| OpType: models.ActionPushTag, | |||
| RefName: "v1.1", | |||
| }, | |||
| }, | |||
| { | |||
| userID: 2, | |||
| repositoryID: 1, | |||
| commitRepoActionOptions: commitRepoActionOptions{ | |||
| PushUpdateOptions: PushUpdateOptions{ | |||
| RefFullName: git.TagPrefix + "v1.1", | |||
| OldCommitID: "oldCommitID", | |||
| NewCommitID: git.EmptySHA, | |||
| }, | |||
| Commits: &repo_module.PushCommits{}, | |||
| }, | |||
| action: models.Action{ | |||
| OpType: models.ActionDeleteTag, | |||
| RefName: "v1.1", | |||
| }, | |||
| }, | |||
| { | |||
| userID: 2, | |||
| repositoryID: 1, | |||
| commitRepoActionOptions: commitRepoActionOptions{ | |||
| PushUpdateOptions: PushUpdateOptions{ | |||
| RefFullName: git.BranchPrefix + "feature/1", | |||
| OldCommitID: "oldCommitID", | |||
| NewCommitID: git.EmptySHA, | |||
| }, | |||
| Commits: &repo_module.PushCommits{}, | |||
| }, | |||
| action: models.Action{ | |||
| OpType: models.ActionDeleteBranch, | |||
| RefName: "feature/1", | |||
| }, | |||
| }, | |||
| } | |||
| for _, s := range samples { | |||
| models.PrepareTestEnv(t) | |||
| user := models.AssertExistsAndLoadBean(t, &models.User{ID: s.userID}).(*models.User) | |||
| repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: s.repositoryID, OwnerID: user.ID}).(*models.Repository) | |||
| repo.Owner = user | |||
| gitRepo, err := git.OpenRepository(repo.RepoPath()) | |||
| assert.NoError(t, err) | |||
| s.commitRepoActionOptions.PusherName = user.Name | |||
| s.commitRepoActionOptions.RepoOwnerID = user.ID | |||
| s.commitRepoActionOptions.RepoName = repo.Name | |||
| s.action.ActUserID = user.ID | |||
| s.action.RepoID = repo.ID | |||
| s.action.Repo = repo | |||
| s.action.IsPrivate = repo.IsPrivate | |||
| testCorrectRepoAction(t, repo, gitRepo, &s.commitRepoActionOptions, &s.action) | |||
| gitRepo.Close() | |||
| } | |||
| } | |||