From 7bfb83e0642530183cc15f3c9208d95f88fdc79a Mon Sep 17 00:00:00 2001 From: zeripath Date: Thu, 26 Dec 2019 11:29:45 +0000 Subject: [PATCH] Batch hook pre- and post-receive calls (#8602) * make notifyWatchers work on multiple actions * more efficient multiple notifyWatchers * Make CommitRepoAction take advantage of multiple actions * Batch post and pre-receive results * Set batch to 30 * Auto adjust timeout & add logging * adjust processing message * Add some messages to pre-receive * Make any non-200 status code from pre-receive an error * Add missing hookPrintResults * Remove shortcut for single action * mistaken merge fix * oops * Move master branch to the front * If repo was empty and the master branch is pushed ensure that that is set as the default branch * fixup * fixup * Missed HookOptions in setdefaultbranch * Batch PushUpdateAddTag and PushUpdateDelTag Co-authored-by: Lunny Xiao --- cmd/hook.go | 217 ++++++++++++---- models/repo_watch.go | 129 ++++++---- models/update.go | 179 +++++++++++++ modules/private/hook.go | 84 ++++-- modules/repofiles/action.go | 188 ++++++++------ modules/repofiles/action_test.go | 4 +- modules/repofiles/update.go | 180 +++++++++++-- modules/repository/repo.go | 6 +- routers/private/hook.go | 427 +++++++++++++++++++------------ routers/private/internal.go | 9 +- 10 files changed, 1023 insertions(+), 400 deletions(-) diff --git a/cmd/hook.go b/cmd/hook.go index 7e4530484..03fa15aab 100644 --- a/cmd/hook.go +++ b/cmd/hook.go @@ -21,6 +21,10 @@ import ( "github.com/urfave/cli" ) +const ( + hookBatchSize = 30 +) + var ( // CmdHook represents the available hooks sub-command. CmdHook = cli.Command{ @@ -75,12 +79,25 @@ Gitea or set your environment appropriately.`, "") prID, _ := strconv.ParseInt(os.Getenv(models.ProtectedBranchPRID), 10, 64) isDeployKey, _ := strconv.ParseBool(os.Getenv(models.EnvIsDeployKey)) - buf := bytes.NewBuffer(nil) + hookOptions := private.HookOptions{ + UserID: userID, + GitAlternativeObjectDirectories: os.Getenv(private.GitAlternativeObjectDirectories), + GitObjectDirectory: os.Getenv(private.GitObjectDirectory), + GitQuarantinePath: os.Getenv(private.GitQuarantinePath), + ProtectedBranchID: prID, + IsDeployKey: isDeployKey, + } + scanner := bufio.NewScanner(os.Stdin) - for scanner.Scan() { - buf.Write(scanner.Bytes()) - buf.WriteByte('\n') + oldCommitIDs := make([]string, hookBatchSize) + newCommitIDs := make([]string, hookBatchSize) + refFullNames := make([]string, hookBatchSize) + count := 0 + total := 0 + lastline := 0 + + for scanner.Scan() { // TODO: support news feeds for wiki if isWiki { continue @@ -94,29 +111,72 @@ Gitea or set your environment appropriately.`, "") oldCommitID := string(fields[0]) newCommitID := string(fields[1]) refFullName := string(fields[2]) + total++ + lastline++ // If the ref is a branch, check if it's protected if strings.HasPrefix(refFullName, git.BranchPrefix) { - statusCode, msg := private.HookPreReceive(username, reponame, private.HookOptions{ - OldCommitID: oldCommitID, - NewCommitID: newCommitID, - RefFullName: refFullName, - UserID: userID, - GitAlternativeObjectDirectories: os.Getenv(private.GitAlternativeObjectDirectories), - GitObjectDirectory: os.Getenv(private.GitObjectDirectory), - GitQuarantinePath: os.Getenv(private.GitQuarantinePath), - ProtectedBranchID: prID, - IsDeployKey: isDeployKey, - }) - switch statusCode { - case http.StatusInternalServerError: - fail("Internal Server Error", msg) - case http.StatusForbidden: - fail(msg, "") + oldCommitIDs[count] = oldCommitID + newCommitIDs[count] = newCommitID + refFullNames[count] = refFullName + count++ + fmt.Fprintf(os.Stdout, "*") + os.Stdout.Sync() + + if count >= hookBatchSize { + fmt.Fprintf(os.Stdout, " Checking %d branches\n", count) + os.Stdout.Sync() + + hookOptions.OldCommitIDs = oldCommitIDs + hookOptions.NewCommitIDs = newCommitIDs + hookOptions.RefFullNames = refFullNames + statusCode, msg := private.HookPreReceive(username, reponame, hookOptions) + switch statusCode { + case http.StatusOK: + // no-op + case http.StatusInternalServerError: + fail("Internal Server Error", msg) + default: + fail(msg, "") + } + count = 0 + lastline = 0 } + } else { + fmt.Fprintf(os.Stdout, ".") + os.Stdout.Sync() + } + if lastline >= hookBatchSize { + fmt.Fprintf(os.Stdout, "\n") + os.Stdout.Sync() + lastline = 0 } } + if count > 0 { + hookOptions.OldCommitIDs = oldCommitIDs[:count] + hookOptions.NewCommitIDs = newCommitIDs[:count] + hookOptions.RefFullNames = refFullNames[:count] + + fmt.Fprintf(os.Stdout, " Checking %d branches\n", count) + os.Stdout.Sync() + + statusCode, msg := private.HookPreReceive(username, reponame, hookOptions) + switch statusCode { + case http.StatusInternalServerError: + fail("Internal Server Error", msg) + case http.StatusForbidden: + fail(msg, "") + } + } else if lastline > 0 { + fmt.Fprintf(os.Stdout, "\n") + os.Stdout.Sync() + lastline = 0 + } + + fmt.Fprintf(os.Stdout, "Checked %d references in total\n", total) + os.Stdout.Sync() + return nil } @@ -156,12 +216,24 @@ Gitea or set your environment appropriately.`, "") pusherID, _ := strconv.ParseInt(os.Getenv(models.EnvPusherID), 10, 64) pusherName := os.Getenv(models.EnvPusherName) - buf := bytes.NewBuffer(nil) + hookOptions := private.HookOptions{ + UserName: pusherName, + UserID: pusherID, + GitAlternativeObjectDirectories: os.Getenv(private.GitAlternativeObjectDirectories), + GitObjectDirectory: os.Getenv(private.GitObjectDirectory), + GitQuarantinePath: os.Getenv(private.GitQuarantinePath), + } + oldCommitIDs := make([]string, hookBatchSize) + newCommitIDs := make([]string, hookBatchSize) + refFullNames := make([]string, hookBatchSize) + count := 0 + total := 0 + wasEmpty := false + masterPushed := false + results := make([]private.HookPostReceiveBranchResult, 0) + scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { - buf.Write(scanner.Bytes()) - buf.WriteByte('\n') - // TODO: support news feeds for wiki if isWiki { continue @@ -172,36 +244,95 @@ Gitea or set your environment appropriately.`, "") continue } - oldCommitID := string(fields[0]) - newCommitID := string(fields[1]) - refFullName := string(fields[2]) + fmt.Fprintf(os.Stdout, ".") + oldCommitIDs[count] = string(fields[0]) + newCommitIDs[count] = string(fields[1]) + refFullNames[count] = string(fields[2]) + if refFullNames[count] == git.BranchPrefix+"master" && newCommitIDs[count] != git.EmptySHA && count == total { + masterPushed = true + } + count++ + total++ + os.Stdout.Sync() + + if count >= hookBatchSize { + fmt.Fprintf(os.Stdout, " Processing %d references\n", count) + os.Stdout.Sync() + hookOptions.OldCommitIDs = oldCommitIDs + hookOptions.NewCommitIDs = newCommitIDs + hookOptions.RefFullNames = refFullNames + resp, err := private.HookPostReceive(repoUser, repoName, hookOptions) + if resp == nil { + hookPrintResults(results) + fail("Internal Server Error", err) + } + wasEmpty = wasEmpty || resp.RepoWasEmpty + results = append(results, resp.Results...) + count = 0 + } + } + + if count == 0 { + if wasEmpty && masterPushed { + // We need to tell the repo to reset the default branch to master + err := private.SetDefaultBranch(repoUser, repoName, "master") + if err != nil { + fail("Internal Server Error", "SetDefaultBranch failed with Error: %v", err) + } + } + fmt.Fprintf(os.Stdout, "Processed %d references in total\n", total) + os.Stdout.Sync() + + hookPrintResults(results) + return nil + } - res, err := private.HookPostReceive(repoUser, repoName, private.HookOptions{ - OldCommitID: oldCommitID, - NewCommitID: newCommitID, - RefFullName: refFullName, - UserID: pusherID, - UserName: pusherName, - }) + hookOptions.OldCommitIDs = oldCommitIDs[:count] + hookOptions.NewCommitIDs = newCommitIDs[:count] + hookOptions.RefFullNames = refFullNames[:count] - if res == nil { - fail("Internal Server Error", err) + fmt.Fprintf(os.Stdout, " Processing %d references\n", count) + os.Stdout.Sync() + + resp, err := private.HookPostReceive(repoUser, repoName, hookOptions) + if resp == nil { + hookPrintResults(results) + fail("Internal Server Error", err) + } + wasEmpty = wasEmpty || resp.RepoWasEmpty + results = append(results, resp.Results...) + + fmt.Fprintf(os.Stdout, "Processed %d references in total\n", total) + os.Stdout.Sync() + + if wasEmpty && masterPushed { + // We need to tell the repo to reset the default branch to master + err := private.SetDefaultBranch(repoUser, repoName, "master") + if err != nil { + fail("Internal Server Error", "SetDefaultBranch failed with Error: %v", err) } + } + + hookPrintResults(results) - if res["message"] == false { + return nil +} + +func hookPrintResults(results []private.HookPostReceiveBranchResult) { + for _, res := range results { + if !res.Message { continue } fmt.Fprintln(os.Stderr, "") - if res["create"] == true { - fmt.Fprintf(os.Stderr, "Create a new pull request for '%s':\n", res["branch"]) - fmt.Fprintf(os.Stderr, " %s\n", res["url"]) + if res.Create { + fmt.Fprintf(os.Stderr, "Create a new pull request for '%s':\n", res.Branch) + fmt.Fprintf(os.Stderr, " %s\n", res.URL) } else { fmt.Fprint(os.Stderr, "Visit the existing pull request:\n") - fmt.Fprintf(os.Stderr, " %s\n", res["url"]) + fmt.Fprintf(os.Stderr, " %s\n", res.URL) } fmt.Fprintln(os.Stderr, "") + os.Stderr.Sync() } - - return nil } diff --git a/models/repo_watch.go b/models/repo_watch.go index 7d421081a..9b3659dbf 100644 --- a/models/repo_watch.go +++ b/models/repo_watch.go @@ -164,68 +164,111 @@ func (repo *Repository) GetWatchers(page int) ([]*User, error) { return users, sess.Find(&users) } -func notifyWatchers(e Engine, act *Action) error { - // Add feeds for user self and all watchers. - watches, err := getWatchers(e, act.RepoID) - if err != nil { - return fmt.Errorf("get watchers: %v", err) - } - - // Add feed for actioner. - act.UserID = act.ActUserID - if _, err = e.InsertOne(act); err != nil { - return fmt.Errorf("insert new actioner: %v", err) - } - - act.loadRepo() - // check repo owner exist. - if err := act.Repo.getOwner(e); err != nil { - return fmt.Errorf("can't get repo owner: %v", err) - } +func notifyWatchers(e Engine, actions ...*Action) error { + var watchers []*Watch + var repo *Repository + var err error + var permCode []bool + var permIssue []bool + var permPR []bool + + for _, act := range actions { + repoChanged := repo == nil || repo.ID != act.RepoID + + if repoChanged { + // Add feeds for user self and all watchers. + watchers, err = getWatchers(e, act.RepoID) + if err != nil { + return fmt.Errorf("get watchers: %v", err) + } + } - // Add feed for organization - if act.Repo.Owner.IsOrganization() && act.ActUserID != act.Repo.Owner.ID { - act.ID = 0 - act.UserID = act.Repo.Owner.ID + // Add feed for actioner. + act.UserID = act.ActUserID if _, err = e.InsertOne(act); err != nil { return fmt.Errorf("insert new actioner: %v", err) } - } - for i := range watches { - if act.ActUserID == watches[i].UserID { - continue + if repoChanged { + act.loadRepo() + repo = act.Repo + + // check repo owner exist. + if err := act.Repo.getOwner(e); err != nil { + return fmt.Errorf("can't get repo owner: %v", err) + } + } else if act.Repo == nil { + act.Repo = repo } - act.ID = 0 - act.UserID = watches[i].UserID - act.Repo.Units = nil + // Add feed for organization + if act.Repo.Owner.IsOrganization() && act.ActUserID != act.Repo.Owner.ID { + act.ID = 0 + act.UserID = act.Repo.Owner.ID + if _, err = e.InsertOne(act); err != nil { + return fmt.Errorf("insert new actioner: %v", err) + } + } - switch act.OpType { - case ActionCommitRepo, ActionPushTag, ActionDeleteTag, ActionDeleteBranch: - if !act.Repo.checkUnitUser(e, act.UserID, false, UnitTypeCode) { - continue + if repoChanged { + permCode = make([]bool, len(watchers)) + permIssue = make([]bool, len(watchers)) + permPR = make([]bool, len(watchers)) + for i, watcher := range watchers { + user, err := getUserByID(e, watcher.UserID) + if err != nil { + permCode[i] = false + permIssue[i] = false + permPR[i] = false + continue + } + perm, err := getUserRepoPermission(e, repo, user) + if err != nil { + permCode[i] = false + permIssue[i] = false + permPR[i] = false + continue + } + permCode[i] = perm.CanRead(UnitTypeCode) + permIssue[i] = perm.CanRead(UnitTypeIssues) + permPR[i] = perm.CanRead(UnitTypePullRequests) } - case ActionCreateIssue, ActionCommentIssue, ActionCloseIssue, ActionReopenIssue: - if !act.Repo.checkUnitUser(e, act.UserID, false, UnitTypeIssues) { + } + + for i, watcher := range watchers { + if act.ActUserID == watcher.UserID { continue } - case ActionCreatePullRequest, ActionCommentPull, ActionMergePullRequest, ActionClosePullRequest, ActionReopenPullRequest: - if !act.Repo.checkUnitUser(e, act.UserID, false, UnitTypePullRequests) { - continue + act.ID = 0 + act.UserID = watcher.UserID + act.Repo.Units = nil + + switch act.OpType { + case ActionCommitRepo, ActionPushTag, ActionDeleteTag, ActionDeleteBranch: + if !permCode[i] { + continue + } + case ActionCreateIssue, ActionCommentIssue, ActionCloseIssue, ActionReopenIssue: + if !permIssue[i] { + continue + } + case ActionCreatePullRequest, ActionCommentPull, ActionMergePullRequest, ActionClosePullRequest, ActionReopenPullRequest: + if !permPR[i] { + continue + } } - } - if _, err = e.InsertOne(act); err != nil { - return fmt.Errorf("insert new action: %v", err) + if _, err = e.InsertOne(act); err != nil { + return fmt.Errorf("insert new action: %v", err) + } } } return nil } // NotifyWatchers creates batch of actions for every watcher. -func NotifyWatchers(act *Action) error { - return notifyWatchers(x, act) +func NotifyWatchers(actions ...*Action) error { + return notifyWatchers(x, actions...) } // NotifyWatchersActions creates batch of actions for every watcher. diff --git a/models/update.go b/models/update.go index deac91b6d..1105c9a82 100644 --- a/models/update.go +++ b/models/update.go @@ -53,6 +53,66 @@ func ListToPushCommits(l *list.List) *PushCommits { return &PushCommits{l.Len(), commits, "", make(map[string]string), make(map[string]*User)} } +// PushUpdateAddDeleteTags updates a number of added and delete tags +func PushUpdateAddDeleteTags(repo *Repository, gitRepo *git.Repository, addTags, delTags []string) error { + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return fmt.Errorf("Unable to begin sess in PushUpdateDeleteTags: %v", err) + } + if err := pushUpdateDeleteTags(sess, repo, delTags); err != nil { + return err + } + if err := pushUpdateAddTags(sess, repo, gitRepo, addTags); err != nil { + return err + } + + return sess.Commit() +} + +// PushUpdateDeleteTags updates a number of delete tags +func PushUpdateDeleteTags(repo *Repository, tags []string) error { + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return fmt.Errorf("Unable to begin sess in PushUpdateDeleteTags: %v", err) + } + if err := pushUpdateDeleteTags(sess, repo, tags); err != nil { + return err + } + + return sess.Commit() +} + +func pushUpdateDeleteTags(e Engine, repo *Repository, tags []string) error { + if len(tags) == 0 { + return nil + } + lowerTags := make([]string, 0, len(tags)) + for _, tag := range tags { + lowerTags = append(lowerTags, strings.ToLower(tag)) + } + + if _, err := e. + Where("repo_id = ? AND is_tag = ?", repo.ID, true). + In("lower_tag_name", lowerTags). + Delete(new(Release)); err != nil { + return fmt.Errorf("Delete: %v", err) + } + + if _, err := e. + Where("repo_id = ? AND is_tag = ?", repo.ID, false). + In("lower_tag_name", lowerTags). + SetExpr("is_draft", true). + SetExpr("num_commits", 0). + SetExpr("sha1", ""). + Update(new(Release)); err != nil { + return fmt.Errorf("Update: %v", err) + } + + return nil +} + // PushUpdateDeleteTag must be called for any push actions to delete tag func PushUpdateDeleteTag(repo *Repository, tagName string) error { rel, err := GetRelease(repo.ID, tagName) @@ -78,6 +138,125 @@ func PushUpdateDeleteTag(repo *Repository, tagName string) error { return nil } +// PushUpdateAddTags updates a number of add tags +func PushUpdateAddTags(repo *Repository, gitRepo *git.Repository, tags []string) error { + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return fmt.Errorf("Unable to begin sess in PushUpdateAddTags: %v", err) + } + if err := pushUpdateAddTags(sess, repo, gitRepo, tags); err != nil { + return err + } + + return sess.Commit() +} +func pushUpdateAddTags(e Engine, repo *Repository, gitRepo *git.Repository, tags []string) error { + if len(tags) == 0 { + return nil + } + + lowerTags := make([]string, 0, len(tags)) + for _, tag := range tags { + lowerTags = append(lowerTags, strings.ToLower(tag)) + } + + releases := make([]Release, 0, len(tags)) + if err := e.Where("repo_id = ?", repo.ID). + In("lower_tag_name", lowerTags).Find(&releases); err != nil { + return fmt.Errorf("GetRelease: %v", err) + } + relMap := make(map[string]*Release) + for _, rel := range releases { + relMap[rel.LowerTagName] = &rel + } + + newReleases := make([]*Release, 0, len(lowerTags)-len(relMap)) + + emailToUser := make(map[string]*User) + + for i, lowerTag := range lowerTags { + tag, err := gitRepo.GetTag(tags[i]) + if err != nil { + return fmt.Errorf("GetTag: %v", err) + } + commit, err := tag.Commit() + if err != nil { + return fmt.Errorf("Commit: %v", err) + } + + sig := tag.Tagger + if sig == nil { + sig = commit.Author + } + if sig == nil { + sig = commit.Committer + } + var author *User + var createdAt = time.Unix(1, 0) + + if sig != nil { + var ok bool + author, ok = emailToUser[sig.Email] + if !ok { + author, err = GetUserByEmail(sig.Email) + if err != nil && !IsErrUserNotExist(err) { + return fmt.Errorf("GetUserByEmail: %v", err) + } + } + createdAt = sig.When + } + + commitsCount, err := commit.CommitsCount() + if err != nil { + return fmt.Errorf("CommitsCount: %v", err) + } + + rel, has := relMap[lowerTag] + + if !has { + rel = &Release{ + RepoID: repo.ID, + Title: "", + TagName: tags[i], + LowerTagName: lowerTag, + Target: "", + Sha1: commit.ID.String(), + NumCommits: commitsCount, + Note: "", + IsDraft: false, + IsPrerelease: false, + IsTag: true, + CreatedUnix: timeutil.TimeStamp(createdAt.Unix()), + } + if author != nil { + rel.PublisherID = author.ID + } + + newReleases = append(newReleases, rel) + } else { + rel.Sha1 = commit.ID.String() + rel.CreatedUnix = timeutil.TimeStamp(createdAt.Unix()) + rel.NumCommits = commitsCount + rel.IsDraft = false + if rel.IsTag && author != nil { + rel.PublisherID = author.ID + } + if _, err = e.ID(rel.ID).AllCols().Update(rel); err != nil { + return fmt.Errorf("Update: %v", err) + } + } + } + + if len(newReleases) > 0 { + if _, err := e.Insert(newReleases); err != nil { + return fmt.Errorf("Insert: %v", err) + } + } + + return nil +} + // PushUpdateAddTag must be called for any push actions to add tag func PushUpdateAddTag(repo *Repository, gitRepo *git.Repository, tagName string) error { rel, err := GetRelease(repo.ID, tagName) diff --git a/modules/private/hook.go b/modules/private/hook.go index cc9703cc7..010fc4d72 100644 --- a/modules/private/hook.go +++ b/modules/private/hook.go @@ -9,6 +9,7 @@ import ( "fmt" "net/http" "net/url" + "time" "code.gitea.io/gitea/modules/setting" ) @@ -22,9 +23,9 @@ const ( // HookOptions represents the options for the Hook calls type HookOptions struct { - OldCommitID string - NewCommitID string - RefFullName string + OldCommitIDs []string + NewCommitIDs []string + RefFullNames []string UserID int64 UserName string GitObjectDirectory string @@ -34,23 +35,33 @@ type HookOptions struct { IsDeployKey bool } +// HookPostReceiveResult represents an individual result from PostReceive +type HookPostReceiveResult struct { + Results []HookPostReceiveBranchResult + RepoWasEmpty bool + Err string +} + +// HookPostReceiveBranchResult represents an individual branch result from PostReceive +type HookPostReceiveBranchResult struct { + Message bool + Create bool + Branch string + URL string +} + // HookPreReceive check whether the provided commits are allowed func HookPreReceive(ownerName, repoName string, opts HookOptions) (int, string) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/pre-receive/%s/%s?old=%s&new=%s&ref=%s&userID=%d&gitObjectDirectory=%s&gitAlternativeObjectDirectories=%s&gitQuarantinePath=%s&prID=%d&isDeployKey=%t", + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/pre-receive/%s/%s", url.PathEscape(ownerName), url.PathEscape(repoName), - url.QueryEscape(opts.OldCommitID), - url.QueryEscape(opts.NewCommitID), - url.QueryEscape(opts.RefFullName), - opts.UserID, - url.QueryEscape(opts.GitObjectDirectory), - url.QueryEscape(opts.GitAlternativeObjectDirectories), - url.QueryEscape(opts.GitQuarantinePath), - opts.ProtectedBranchID, - opts.IsDeployKey, ) - - resp, err := newInternalRequest(reqURL, "GET").Response() + req := newInternalRequest(reqURL, "POST") + req = req.Header("Content-Type", "application/json") + jsonBytes, _ := json.Marshal(opts) + req.Body(jsonBytes) + req.SetTimeout(60*time.Second, time.Duration(60+len(opts.OldCommitIDs))*time.Second) + resp, err := req.Response() if err != nil { return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error()) } @@ -64,17 +75,18 @@ func HookPreReceive(ownerName, repoName string, opts HookOptions) (int, string) } // HookPostReceive updates services and users -func HookPostReceive(ownerName, repoName string, opts HookOptions) (map[string]interface{}, string) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/post-receive/%s/%s?old=%s&new=%s&ref=%s&userID=%d&username=%s", +func HookPostReceive(ownerName, repoName string, opts HookOptions) (*HookPostReceiveResult, string) { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/post-receive/%s/%s", url.PathEscape(ownerName), url.PathEscape(repoName), - url.QueryEscape(opts.OldCommitID), - url.QueryEscape(opts.NewCommitID), - url.QueryEscape(opts.RefFullName), - opts.UserID, - url.QueryEscape(opts.UserName)) + ) - resp, err := newInternalRequest(reqURL, "GET").Response() + req := newInternalRequest(reqURL, "POST") + req = req.Header("Content-Type", "application/json") + req.SetTimeout(60*time.Second, time.Duration(60+len(opts.OldCommitIDs))*time.Second) + jsonBytes, _ := json.Marshal(opts) + req.Body(jsonBytes) + resp, err := req.Response() if err != nil { return nil, fmt.Sprintf("Unable to contact gitea: %v", err.Error()) } @@ -83,8 +95,30 @@ func HookPostReceive(ownerName, repoName string, opts HookOptions) (map[string]i if resp.StatusCode != http.StatusOK { return nil, decodeJSONError(resp).Err } - res := map[string]interface{}{} - _ = json.NewDecoder(resp.Body).Decode(&res) + res := &HookPostReceiveResult{} + _ = json.NewDecoder(resp.Body).Decode(res) return res, "" } + +// SetDefaultBranch will set the default branch to the provided branch for the provided repository +func SetDefaultBranch(ownerName, repoName, branch string) error { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/set-default-branch/%s/%s/%s", + url.PathEscape(ownerName), + url.PathEscape(repoName), + url.PathEscape(branch), + ) + req := newInternalRequest(reqURL, "POST") + req = req.Header("Content-Type", "application/json") + + req.SetTimeout(60*time.Second, 60*time.Second) + resp, err := req.Response() + if err != nil { + return fmt.Errorf("Unable to contact gitea: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("Error returned from gitea: %v", decodeJSONError(resp).Err) + } + return nil +} diff --git a/modules/repofiles/action.go b/modules/repofiles/action.go index a5a5e151c..d20724711 100644 --- a/modules/repofiles/action.go +++ b/modules/repofiles/action.go @@ -159,112 +159,132 @@ type CommitRepoActionOptions struct { // CommitRepoAction adds new commit action to the repository, and prepare // corresponding webhooks. -func CommitRepoAction(opts CommitRepoActionOptions) error { - pusher, err := models.GetUserByName(opts.PusherName) - if err != nil { - return fmt.Errorf("GetUserByName [%s]: %v", opts.PusherName, err) - } - - repo, err := models.GetRepositoryByName(opts.RepoOwnerID, opts.RepoName) - if err != nil { - return fmt.Errorf("GetRepositoryByName [owner_id: %d, name: %s]: %v", opts.RepoOwnerID, opts.RepoName, err) - } - - refName := git.RefEndName(opts.RefFullName) +func CommitRepoAction(optsList ...*CommitRepoActionOptions) error { + var pusher *models.User + var repo *models.Repository + actions := make([]*models.Action, len(optsList)) + + for i, opts := range optsList { + if pusher == nil || pusher.Name != opts.PusherName { + var err error + pusher, err = models.GetUserByName(opts.PusherName) + if err != nil { + return fmt.Errorf("GetUserByName [%s]: %v", opts.PusherName, err) + } + } - // Change default branch and empty status only if pushed ref is non-empty branch. - if repo.IsEmpty && opts.NewCommitID != git.EmptySHA && strings.HasPrefix(opts.RefFullName, git.BranchPrefix) { - repo.DefaultBranch = refName - repo.IsEmpty = false - if refName != "master" { - gitRepo, err := git.OpenRepository(repo.RepoPath()) + if repo == nil || repo.OwnerID != opts.RepoOwnerID || repo.Name != opts.RepoName { + var err error + if repo != nil { + // Change repository empty status and update last updated time. + if err := models.UpdateRepository(repo, false); err != nil { + return fmt.Errorf("UpdateRepository: %v", err) + } + } + repo, err = models.GetRepositoryByName(opts.RepoOwnerID, opts.RepoName) if err != nil { - return err + return fmt.Errorf("GetRepositoryByName [owner_id: %d, name: %s]: %v", opts.RepoOwnerID, opts.RepoName, err) } - if err := gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil { - if !git.IsErrUnsupportedVersion(err) { - gitRepo.Close() + } + refName := git.RefEndName(opts.RefFullName) + + // Change default branch and empty status only if pushed ref is non-empty branch. + if repo.IsEmpty && opts.NewCommitID != git.EmptySHA && strings.HasPrefix(opts.RefFullName, git.BranchPrefix) { + repo.DefaultBranch = refName + repo.IsEmpty = false + if refName != "master" { + gitRepo, err := git.OpenRepository(repo.RepoPath()) + if err != nil { return err } + if err := gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil { + if !git.IsErrUnsupportedVersion(err) { + gitRepo.Close() + return err + } + } + gitRepo.Close() } - gitRepo.Close() } - } - // Change repository empty status and update last updated time. - if err = models.UpdateRepository(repo, false); err != nil { - return fmt.Errorf("UpdateRepository: %v", err) - } + isNewBranch := false + opType := models.ActionCommitRepo - isNewBranch := false - opType := models.ActionCommitRepo - // Check it's tag push or branch. - if strings.HasPrefix(opts.RefFullName, git.TagPrefix) { - opType = models.ActionPushTag - if opts.NewCommitID == git.EmptySHA { - opType = models.ActionDeleteTag - } - opts.Commits = &models.PushCommits{} - } else if opts.NewCommitID == git.EmptySHA { - opType = models.ActionDeleteBranch - opts.Commits = &models.PushCommits{} - } else { - // if not the first commit, set the compare URL. - if opts.OldCommitID == git.EmptySHA { - isNewBranch = true + // Check it's tag push or branch. + if strings.HasPrefix(opts.RefFullName, git.TagPrefix) { + opType = models.ActionPushTag + if opts.NewCommitID == git.EmptySHA { + opType = models.ActionDeleteTag + } + opts.Commits = &models.PushCommits{} + } else if opts.NewCommitID == git.EmptySHA { + opType = models.ActionDeleteBranch + opts.Commits = &models.PushCommits{} } else { - opts.Commits.CompareURL = repo.ComposeCompareURL(opts.OldCommitID, opts.NewCommitID) - } + // if not the first commit, set the compare URL. + if opts.OldCommitID == git.EmptySHA { + isNewBranch = true + } else { + opts.Commits.CompareURL = repo.ComposeCompareURL(opts.OldCommitID, opts.NewCommitID) + } - if err = UpdateIssuesCommit(pusher, repo, opts.Commits.Commits, refName); err != nil { - log.Error("updateIssuesCommit: %v", err) + if err := UpdateIssuesCommit(pusher, repo, opts.Commits.Commits, refName); err != nil { + log.Error("updateIssuesCommit: %v", err) + } } - } - if len(opts.Commits.Commits) > setting.UI.FeedMaxCommitNum { - opts.Commits.Commits = opts.Commits.Commits[:setting.UI.FeedMaxCommitNum] - } - - data, err := json.Marshal(opts.Commits) - if err != nil { - return fmt.Errorf("Marshal: %v", err) - } + if len(opts.Commits.Commits) > setting.UI.FeedMaxCommitNum { + opts.Commits.Commits = opts.Commits.Commits[:setting.UI.FeedMaxCommitNum] + } - if err = models.NotifyWatchers(&models.Action{ - ActUserID: pusher.ID, - ActUser: pusher, - OpType: opType, - Content: string(data), - RepoID: repo.ID, - Repo: repo, - RefName: refName, - IsPrivate: repo.IsPrivate, - }); err != nil { - return fmt.Errorf("NotifyWatchers: %v", err) - } + data, err := json.Marshal(opts.Commits) + if err != nil { + return fmt.Errorf("Marshal: %v", err) + } - var isHookEventPush = true - switch opType { - case models.ActionCommitRepo: // Push - if isNewBranch { - notification.NotifyCreateRef(pusher, repo, "branch", opts.RefFullName) + actions[i] = &models.Action{ + ActUserID: pusher.ID, + ActUser: pusher, + OpType: opType, + Content: string(data), + RepoID: repo.ID, + Repo: repo, + RefName: refName, + IsPrivate: repo.IsPrivate, } - case models.ActionDeleteBranch: // Delete Branch - notification.NotifyDeleteRef(pusher, repo, "branch", opts.RefFullName) + var isHookEventPush = true + switch opType { + case models.ActionCommitRepo: // Push + if isNewBranch { + notification.NotifyCreateRef(pusher, repo, "branch", opts.RefFullName) + } + case models.ActionDeleteBranch: // Delete Branch + notification.NotifyDeleteRef(pusher, repo, "branch", opts.RefFullName) + + case models.ActionPushTag: // Create + notification.NotifyCreateRef(pusher, repo, "tag", opts.RefFullName) - case models.ActionPushTag: // Create - notification.NotifyCreateRef(pusher, repo, "tag", opts.RefFullName) + case models.ActionDeleteTag: // Delete Tag + notification.NotifyDeleteRef(pusher, repo, "tag", opts.RefFullName) + default: + isHookEventPush = false + } - case models.ActionDeleteTag: // Delete Tag - notification.NotifyDeleteRef(pusher, repo, "tag", opts.RefFullName) - default: - isHookEventPush = false + if isHookEventPush { + notification.NotifyPushCommits(pusher, repo, opts.RefFullName, opts.OldCommitID, opts.NewCommitID, opts.Commits) + } } - if isHookEventPush { - notification.NotifyPushCommits(pusher, repo, opts.RefFullName, opts.OldCommitID, opts.NewCommitID, opts.Commits) + if repo != nil { + // Change repository empty status and update last updated time. + if err := models.UpdateRepository(repo, false); err != nil { + return fmt.Errorf("UpdateRepository: %v", err) + } } + if err := models.NotifyWatchers(actions...); err != nil { + return fmt.Errorf("NotifyWatchers: %v", err) + } return nil } diff --git a/modules/repofiles/action_test.go b/modules/repofiles/action_test.go index 5a4c6231f..97ac1c45e 100644 --- a/modules/repofiles/action_test.go +++ b/modules/repofiles/action_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/assert" ) -func testCorrectRepoAction(t *testing.T, opts CommitRepoActionOptions, actionBean *models.Action) { +func testCorrectRepoAction(t *testing.T, opts *CommitRepoActionOptions, actionBean *models.Action) { models.AssertNotExistsBean(t, actionBean) assert.NoError(t, CommitRepoAction(opts)) models.AssertExistsAndLoadBean(t, actionBean) @@ -121,7 +121,7 @@ func TestCommitRepoAction(t *testing.T) { s.action.Repo = repo s.action.IsPrivate = repo.IsPrivate - testCorrectRepoAction(t, s.commitRepoActionOptions, &s.action) + testCorrectRepoAction(t, &s.commitRepoActionOptions, &s.action) } } diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index 8a95b4422..c97d3d46e 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -432,6 +432,7 @@ type PushUpdateOptions struct { RefFullName string OldCommitID string NewCommitID string + Branch string } // PushUpdate must be called for any push actions in order to @@ -460,21 +461,161 @@ func PushUpdate(repo *models.Repository, branch string, opts PushUpdateOptions) log.Error("Failed to update size for repository: %v", err) } + commitRepoActionOptions, err := createCommitRepoActionOption(repo, gitRepo, &opts) + if err != nil { + return err + } + + if err := CommitRepoAction(commitRepoActionOptions); err != nil { + return fmt.Errorf("CommitRepoAction: %v", err) + } + + pusher, err := models.GetUserByID(opts.PusherID) + if err != nil { + return err + } + + log.Trace("TriggerTask '%s/%s' by %s", repo.Name, branch, pusher.Name) + + go pull_service.AddTestPullRequestTask(pusher, repo.ID, branch, true) + + if err = models.WatchIfAuto(opts.PusherID, repo.ID, true); err != nil { + log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err) + } + + return nil +} + +// PushUpdates generates push action history feeds for push updating multiple refs +func PushUpdates(repo *models.Repository, optsList []*PushUpdateOptions) error { + repoPath := repo.RepoPath() + _, err := git.NewCommand("update-server-info").RunInDir(repoPath) + if err != nil { + return fmt.Errorf("Failed to call 'git update-server-info': %v", err) + } + gitRepo, err := git.OpenRepository(repoPath) + if err != nil { + return fmt.Errorf("OpenRepository: %v", err) + } + if err = repo.UpdateSize(); err != nil { + log.Error("Failed to update size for repository: %v", err) + } + + actions, err := createCommitRepoActions(repo, gitRepo, optsList) + if err != nil { + return err + } + if err := CommitRepoAction(actions...); err != nil { + return fmt.Errorf("CommitRepoAction: %v", err) + } + + var pusher *models.User + + for _, opts := range optsList { + if pusher == nil || pusher.ID != opts.PusherID { + var err error + pusher, err = models.GetUserByID(opts.PusherID) + if err != nil { + return err + } + } + + log.Trace("TriggerTask '%s/%s' by %s", repo.Name, opts.Branch, pusher.Name) + + go pull_service.AddTestPullRequestTask(pusher, repo.ID, opts.Branch, true) + + if err = models.WatchIfAuto(opts.PusherID, repo.ID, true); err != nil { + log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err) + } + } + + return nil +} + +func createCommitRepoActions(repo *models.Repository, gitRepo *git.Repository, optsList []*PushUpdateOptions) ([]*CommitRepoActionOptions, error) { + addTags := make([]string, 0, len(optsList)) + delTags := make([]string, 0, len(optsList)) + actions := make([]*CommitRepoActionOptions, 0, len(optsList)) + + for _, opts := range optsList { + isNewRef := opts.OldCommitID == git.EmptySHA + isDelRef := opts.NewCommitID == git.EmptySHA + if isNewRef && isDelRef { + return nil, fmt.Errorf("Old and new revisions are both %s", git.EmptySHA) + } + var commits = &models.PushCommits{} + if strings.HasPrefix(opts.RefFullName, git.TagPrefix) { + // If is tag reference + tagName := opts.RefFullName[len(git.TagPrefix):] + if isDelRef { + delTags = append(delTags, tagName) + } else { + cache.Remove(repo.GetCommitsCountCacheKey(tagName, true)) + addTags = append(addTags, tagName) + } + } else if !isDelRef { + // If is branch reference + + // Clear cache for branch commit count + cache.Remove(repo.GetCommitsCountCacheKey(opts.RefFullName[len(git.BranchPrefix):], true)) + + newCommit, err := gitRepo.GetCommit(opts.NewCommitID) + if err != nil { + return nil, fmt.Errorf("gitRepo.GetCommit: %v", err) + } + + // Push new branch. + var l *list.List + if isNewRef { + l, err = newCommit.CommitsBeforeLimit(10) + if err != nil { + return nil, fmt.Errorf("newCommit.CommitsBeforeLimit: %v", err) + } + } else { + l, err = newCommit.CommitsBeforeUntil(opts.OldCommitID) + if err != nil { + return nil, fmt.Errorf("newCommit.CommitsBeforeUntil: %v", err) + } + } + + commits = models.ListToPushCommits(l) + } + actions = append(actions, &CommitRepoActionOptions{ + PusherName: opts.PusherName, + RepoOwnerID: repo.OwnerID, + RepoName: repo.Name, + RefFullName: opts.RefFullName, + OldCommitID: opts.OldCommitID, + NewCommitID: opts.NewCommitID, + Commits: commits, + }) + } + if err := models.PushUpdateAddDeleteTags(repo, gitRepo, addTags, delTags); err != nil { + return nil, fmt.Errorf("PushUpdateAddDeleteTags: %v", err) + } + return actions, nil +} + +func createCommitRepoActionOption(repo *models.Repository, gitRepo *git.Repository, opts *PushUpdateOptions) (*CommitRepoActionOptions, error) { + isNewRef := opts.OldCommitID == git.EmptySHA + isDelRef := opts.NewCommitID == git.EmptySHA + if isNewRef && isDelRef { + return nil, fmt.Errorf("Old and new revisions are both %s", git.EmptySHA) + } + var commits = &models.PushCommits{} if strings.HasPrefix(opts.RefFullName, git.TagPrefix) { // If is tag reference tagName := opts.RefFullName[len(git.TagPrefix):] if isDelRef { - err = models.PushUpdateDeleteTag(repo, tagName) - if err != nil { - return fmt.Errorf("PushUpdateDeleteTag: %v", err) + if err := models.PushUpdateDeleteTag(repo, tagName); err != nil { + return nil, fmt.Errorf("PushUpdateDeleteTag: %v", err) } } else { // Clear cache for tag commit count cache.Remove(repo.GetCommitsCountCacheKey(tagName, true)) - err = models.PushUpdateAddTag(repo, gitRepo, tagName) - if err != nil { - return fmt.Errorf("PushUpdateAddTag: %v", err) + if err := models.PushUpdateAddTag(repo, gitRepo, tagName); err != nil { + return nil, fmt.Errorf("PushUpdateAddTag: %v", err) } } } else if !isDelRef { @@ -485,7 +626,7 @@ func PushUpdate(repo *models.Repository, branch string, opts PushUpdateOptions) newCommit, err := gitRepo.GetCommit(opts.NewCommitID) if err != nil { - return fmt.Errorf("gitRepo.GetCommit: %v", err) + return nil, fmt.Errorf("gitRepo.GetCommit: %v", err) } // Push new branch. @@ -493,19 +634,19 @@ func PushUpdate(repo *models.Repository, branch string, opts PushUpdateOptions) if isNewRef { l, err = newCommit.CommitsBeforeLimit(10) if err != nil { - return fmt.Errorf("newCommit.CommitsBeforeLimit: %v", err) + return nil, fmt.Errorf("newCommit.CommitsBeforeLimit: %v", err) } } else { l, err = newCommit.CommitsBeforeUntil(opts.OldCommitID) if err != nil { - return fmt.Errorf("newCommit.CommitsBeforeUntil: %v", err) + return nil, fmt.Errorf("newCommit.CommitsBeforeUntil: %v", err) } } commits = models.ListToPushCommits(l) } - if err := CommitRepoAction(CommitRepoActionOptions{ + return &CommitRepoActionOptions{ PusherName: opts.PusherName, RepoOwnerID: repo.OwnerID, RepoName: repo.Name, @@ -513,22 +654,5 @@ func PushUpdate(repo *models.Repository, branch string, opts PushUpdateOptions) OldCommitID: opts.OldCommitID, NewCommitID: opts.NewCommitID, Commits: commits, - }); err != nil { - return fmt.Errorf("CommitRepoAction: %v", err) - } - - pusher, err := models.GetUserByID(opts.PusherID) - if err != nil { - return err - } - - log.Trace("TriggerTask '%s/%s' by %s", repo.Name, branch, pusher.Name) - - go pull_service.AddTestPullRequestTask(pusher, repo.ID, branch, true) - - if err = models.WatchIfAuto(opts.PusherID, repo.ID, true); err != nil { - log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err) - } - - return nil + }, nil } diff --git a/modules/repository/repo.go b/modules/repository/repo.go index ea526a1e3..9351ab397 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -197,11 +197,11 @@ func SyncReleasesWithTags(repo *models.Repository, gitRepo *git.Repository) erro } commitID, err := gitRepo.GetTagCommitID(rel.TagName) if err != nil && !git.IsErrNotExist(err) { - return fmt.Errorf("GetTagCommitID: %v", err) + return fmt.Errorf("GetTagCommitID: %s: %v", rel.TagName, err) } if git.IsErrNotExist(err) || commitID != rel.Sha1 { if err := models.PushUpdateDeleteTag(repo, rel.TagName); err != nil { - return fmt.Errorf("PushUpdateDeleteTag: %v", err) + return fmt.Errorf("PushUpdateDeleteTag: %s: %v", rel.TagName, err) } } else { existingRelTags[strings.ToLower(rel.TagName)] = struct{}{} @@ -215,7 +215,7 @@ func SyncReleasesWithTags(repo *models.Repository, gitRepo *git.Repository) erro for _, tagName := range tags { if _, ok := existingRelTags[strings.ToLower(tagName)]; !ok { if err := models.PushUpdateAddTag(repo, gitRepo, tagName); err != nil { - return fmt.Errorf("pushUpdateAddTag: %v", err) + return fmt.Errorf("pushUpdateAddTag: %s: %v", tagName, err) } } } diff --git a/routers/private/hook.go b/routers/private/hook.go index 2644302ea..dc5001ad4 100644 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -22,20 +22,9 @@ import ( ) // HookPreReceive checks whether a individual commit is acceptable -func HookPreReceive(ctx *macaron.Context) { +func HookPreReceive(ctx *macaron.Context, opts private.HookOptions) { ownerName := ctx.Params(":owner") repoName := ctx.Params(":repo") - oldCommitID := ctx.QueryTrim("old") - newCommitID := ctx.QueryTrim("new") - refFullName := ctx.QueryTrim("ref") - userID := ctx.QueryInt64("userID") - gitObjectDirectory := ctx.QueryTrim("gitObjectDirectory") - gitAlternativeObjectDirectories := ctx.QueryTrim("gitAlternativeObjectDirectories") - gitQuarantinePath := ctx.QueryTrim("gitQuarantinePath") - prID := ctx.QueryInt64("prID") - isDeployKey := ctx.QueryBool("isDeployKey") - - branchName := strings.TrimPrefix(refFullName, git.BranchPrefix) repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName) if err != nil { log.Error("Unable to get repository: %s/%s Error: %v", ownerName, repoName, err) @@ -45,206 +34,304 @@ func HookPreReceive(ctx *macaron.Context) { return } repo.OwnerName = ownerName - protectBranch, err := models.GetProtectedBranchBy(repo.ID, branchName) - if err != nil { - log.Error("Unable to get protected branch: %s in %-v Error: %v", branchName, repo, err) - ctx.JSON(500, map[string]interface{}{ - "err": err.Error(), - }) - return - } - if protectBranch != nil && protectBranch.IsProtected() { - // check and deletion - if newCommitID == git.EmptySHA { - log.Warn("Forbidden: Branch: %s in %-v is protected from deletion", branchName, repo) - ctx.JSON(http.StatusForbidden, map[string]interface{}{ - "err": fmt.Sprintf("branch %s is protected from deletion", branchName), + + for i := range opts.OldCommitIDs { + oldCommitID := opts.OldCommitIDs[i] + newCommitID := opts.NewCommitIDs[i] + refFullName := opts.RefFullNames[i] + + branchName := strings.TrimPrefix(refFullName, git.BranchPrefix) + protectBranch, err := models.GetProtectedBranchBy(repo.ID, branchName) + if err != nil { + log.Error("Unable to get protected branch: %s in %-v Error: %v", branchName, repo, err) + ctx.JSON(500, map[string]interface{}{ + "err": err.Error(), }) return } - - // detect force push - if git.EmptySHA != oldCommitID { - env := os.Environ() - if gitAlternativeObjectDirectories != "" { - env = append(env, - private.GitAlternativeObjectDirectories+"="+gitAlternativeObjectDirectories) - } - if gitObjectDirectory != "" { - env = append(env, - private.GitObjectDirectory+"="+gitObjectDirectory) - } - if gitQuarantinePath != "" { - env = append(env, - private.GitQuarantinePath+"="+gitQuarantinePath) - } - - output, err := git.NewCommand("rev-list", "--max-count=1", oldCommitID, "^"+newCommitID).RunInDirWithEnv(repo.RepoPath(), env) - if err != nil { - log.Error("Unable to detect force push between: %s and %s in %-v Error: %v", oldCommitID, newCommitID, repo, err) - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": fmt.Sprintf("Fail to detect force push: %v", err), - }) - return - } else if len(output) > 0 { - log.Warn("Forbidden: Branch: %s in %-v is protected from force push", branchName, repo) + if protectBranch != nil && protectBranch.IsProtected() { + // check and deletion + if newCommitID == git.EmptySHA { + log.Warn("Forbidden: Branch: %s in %-v is protected from deletion", branchName, repo) ctx.JSON(http.StatusForbidden, map[string]interface{}{ - "err": fmt.Sprintf("branch %s is protected from force push", branchName), + "err": fmt.Sprintf("branch %s is protected from deletion", branchName), }) return - } - } - canPush := false - if isDeployKey { - canPush = protectBranch.CanPush && (!protectBranch.EnableWhitelist || protectBranch.WhitelistDeployKeys) - } else { - canPush = protectBranch.CanUserPush(userID) - } - if !canPush && prID > 0 { - pr, err := models.GetPullRequestByID(prID) - if err != nil { - log.Error("Unable to get PullRequest %d Error: %v", prID, err) - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": fmt.Sprintf("Unable to get PullRequest %d Error: %v", prID, err), - }) - return + // detect force push + if git.EmptySHA != oldCommitID { + env := os.Environ() + if opts.GitAlternativeObjectDirectories != "" { + env = append(env, + private.GitAlternativeObjectDirectories+"="+opts.GitAlternativeObjectDirectories) + } + if opts.GitObjectDirectory != "" { + env = append(env, + private.GitObjectDirectory+"="+opts.GitObjectDirectory) + } + if opts.GitQuarantinePath != "" { + env = append(env, + private.GitQuarantinePath+"="+opts.GitQuarantinePath) + } + + output, err := git.NewCommand("rev-list", "--max-count=1", oldCommitID, "^"+newCommitID).RunInDirWithEnv(repo.RepoPath(), env) + if err != nil { + log.Error("Unable to detect force push between: %s and %s in %-v Error: %v", oldCommitID, newCommitID, repo, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "err": fmt.Sprintf("Fail to detect force push: %v", err), + }) + return + } else if len(output) > 0 { + log.Warn("Forbidden: Branch: %s in %-v is protected from force push", branchName, repo) + ctx.JSON(http.StatusForbidden, map[string]interface{}{ + "err": fmt.Sprintf("branch %s is protected from force push", branchName), + }) + return + + } + } + canPush := false + if opts.IsDeployKey { + canPush = protectBranch.CanPush && (!protectBranch.EnableWhitelist || protectBranch.WhitelistDeployKeys) + } else { + canPush = protectBranch.CanUserPush(opts.UserID) } - if !protectBranch.HasEnoughApprovals(pr) { - log.Warn("Forbidden: User %d cannot push to protected branch: %s in %-v and pr #%d does not have enough approvals", userID, branchName, repo, pr.Index) + if !canPush && opts.ProtectedBranchID > 0 { + pr, err := models.GetPullRequestByID(opts.ProtectedBranchID) + if err != nil { + log.Error("Unable to get PullRequest %d Error: %v", opts.ProtectedBranchID, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "err": fmt.Sprintf("Unable to get PullRequest %d Error: %v", opts.ProtectedBranchID, err), + }) + return + } + if !protectBranch.HasEnoughApprovals(pr) { + log.Warn("Forbidden: User %d cannot push to protected branch: %s in %-v and pr #%d does not have enough approvals", opts.UserID, branchName, repo, pr.Index) + ctx.JSON(http.StatusForbidden, map[string]interface{}{ + "err": fmt.Sprintf("protected branch %s can not be pushed to and pr #%d does not have enough approvals", branchName, opts.ProtectedBranchID), + }) + return + } + } else if !canPush { + log.Warn("Forbidden: User %d cannot push to protected branch: %s in %-v", opts.UserID, branchName, repo) ctx.JSON(http.StatusForbidden, map[string]interface{}{ - "err": fmt.Sprintf("protected branch %s can not be pushed to and pr #%d does not have enough approvals", branchName, prID), + "err": fmt.Sprintf("protected branch %s can not be pushed to", branchName), }) return } - } else if !canPush { - log.Warn("Forbidden: User %d cannot push to protected branch: %s in %-v", userID, branchName, repo) - ctx.JSON(http.StatusForbidden, map[string]interface{}{ - "err": fmt.Sprintf("protected branch %s can not be pushed to", branchName), - }) - return } } + ctx.PlainText(http.StatusOK, []byte("ok")) } // HookPostReceive updates services and users -func HookPostReceive(ctx *macaron.Context) { +func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { ownerName := ctx.Params(":owner") repoName := ctx.Params(":repo") - oldCommitID := ctx.Query("old") - newCommitID := ctx.Query("new") - refFullName := ctx.Query("ref") - userID := ctx.QueryInt64("userID") - userName := ctx.Query("username") - - branch := refFullName - if strings.HasPrefix(refFullName, git.BranchPrefix) { - branch = strings.TrimPrefix(refFullName, git.BranchPrefix) - } else if strings.HasPrefix(refFullName, git.TagPrefix) { - branch = strings.TrimPrefix(refFullName, git.TagPrefix) - } - // Only trigger activity updates for changes to branches or - // tags. Updates to other refs (eg, refs/notes, refs/changes, - // or other less-standard refs spaces are ignored since there - // may be a very large number of them). - if strings.HasPrefix(refFullName, git.BranchPrefix) || strings.HasPrefix(refFullName, git.TagPrefix) { - repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName) - if err != nil { - log.Error("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err) - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": fmt.Sprintf("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err), - }) - return + var repo *models.Repository + updates := make([]*repofiles.PushUpdateOptions, 0, len(opts.OldCommitIDs)) + wasEmpty := false + + for i := range opts.OldCommitIDs { + refFullName := opts.RefFullNames[i] + branch := opts.RefFullNames[i] + if strings.HasPrefix(branch, git.BranchPrefix) { + branch = strings.TrimPrefix(branch, git.BranchPrefix) + } else { + branch = strings.TrimPrefix(branch, git.TagPrefix) } - if err := repofiles.PushUpdate(repo, branch, repofiles.PushUpdateOptions{ - RefFullName: refFullName, - OldCommitID: oldCommitID, - NewCommitID: newCommitID, - PusherID: userID, - PusherName: userName, - RepoUserName: ownerName, - RepoName: repoName, - }); err != nil { - log.Error("Failed to Update: %s/%s Branch: %s Error: %v", ownerName, repoName, branch, err) - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": fmt.Sprintf("Failed to Update: %s/%s Branch: %s Error: %v", ownerName, repoName, branch, err), - }) - return + + // Only trigger activity updates for changes to branches or + // tags. Updates to other refs (eg, refs/notes, refs/changes, + // or other less-standard refs spaces are ignored since there + // may be a very large number of them). + if strings.HasPrefix(refFullName, git.BranchPrefix) || strings.HasPrefix(refFullName, git.TagPrefix) { + if repo == nil { + var err error + repo, err = models.GetRepositoryByOwnerAndName(ownerName, repoName) + if err != nil { + log.Error("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err) + ctx.JSON(http.StatusInternalServerError, private.HookPostReceiveResult{ + Err: fmt.Sprintf("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err), + }) + return + } + if repo.OwnerName == "" { + repo.OwnerName = ownerName + } + wasEmpty = repo.IsEmpty + } + + option := repofiles.PushUpdateOptions{ + RefFullName: refFullName, + OldCommitID: opts.OldCommitIDs[i], + NewCommitID: opts.NewCommitIDs[i], + Branch: branch, + PusherID: opts.UserID, + PusherName: opts.UserName, + RepoUserName: ownerName, + RepoName: repoName, + } + updates = append(updates, &option) + if repo.IsEmpty && branch == "master" && strings.HasPrefix(refFullName, git.BranchPrefix) { + // put the master branch first + copy(updates[1:], updates) + updates[0] = &option + } } } - if newCommitID != git.EmptySHA && strings.HasPrefix(refFullName, git.BranchPrefix) { - repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName) - if err != nil { - log.Error("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err) - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": fmt.Sprintf("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err), - }) - return - } - repo.OwnerName = ownerName + if repo != nil && len(updates) > 0 { + if err := repofiles.PushUpdates(repo, updates); err != nil { + log.Error("Failed to Update: %s/%s Total Updates: %d", ownerName, repoName, len(updates)) + for i, update := range updates { + log.Error("Failed to Update: %s/%s Update: %d/%d: Branch: %s", ownerName, repoName, i, len(updates), update.Branch) + } + log.Error("Failed to Update: %s/%s Error: %v", ownerName, repoName, err) - pullRequestAllowed := repo.AllowsPulls() - if !pullRequestAllowed { - ctx.JSON(http.StatusOK, map[string]interface{}{ - "message": false, + ctx.JSON(http.StatusInternalServerError, private.HookPostReceiveResult{ + Err: fmt.Sprintf("Failed to Update: %s/%s Error: %v", ownerName, repoName, err), }) return } + } + + results := make([]private.HookPostReceiveBranchResult, 0, len(opts.OldCommitIDs)) + + // We have to reload the repo in case its state is changed above + repo = nil + var baseRepo *models.Repository + + for i := range opts.OldCommitIDs { + refFullName := opts.RefFullNames[i] + newCommitID := opts.NewCommitIDs[i] + + branch := git.RefEndName(opts.RefFullNames[i]) + + if newCommitID != git.EmptySHA && strings.HasPrefix(refFullName, git.BranchPrefix) { + if repo == nil { + var err error + repo, err = models.GetRepositoryByOwnerAndName(ownerName, repoName) + if err != nil { + log.Error("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err) + ctx.JSON(http.StatusInternalServerError, private.HookPostReceiveResult{ + Err: fmt.Sprintf("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err), + RepoWasEmpty: wasEmpty, + }) + return + } + if repo.OwnerName == "" { + repo.OwnerName = ownerName + } + + if !repo.AllowsPulls() { + // We can stop there's no need to go any further + ctx.JSON(http.StatusOK, private.HookPostReceiveResult{ + RepoWasEmpty: wasEmpty, + }) + return + } + baseRepo = repo + + if repo.IsFork { + if err := repo.GetBaseRepo(); err != nil { + log.Error("Failed to get Base Repository of Forked repository: %-v Error: %v", repo, err) + ctx.JSON(http.StatusInternalServerError, private.HookPostReceiveResult{ + Err: fmt.Sprintf("Failed to get Base Repository of Forked repository: %-v Error: %v", repo, err), + RepoWasEmpty: wasEmpty, + }) + return + } + baseRepo = repo.BaseRepo + } + } + + if !repo.IsFork && branch == baseRepo.DefaultBranch { + results = append(results, private.HookPostReceiveBranchResult{}) + continue + } - baseRepo := repo - if repo.IsFork { - if err := repo.GetBaseRepo(); err != nil { - log.Error("Failed to get Base Repository of Forked repository: %-v Error: %v", repo, err) - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": fmt.Sprintf("Failed to get Base Repository of Forked repository: %-v Error: %v", repo, err), + pr, err := models.GetUnmergedPullRequest(repo.ID, baseRepo.ID, branch, baseRepo.DefaultBranch) + if err != nil && !models.IsErrPullRequestNotExist(err) { + log.Error("Failed to get active PR in: %-v Branch: %s to: %-v Branch: %s Error: %v", repo, branch, baseRepo, baseRepo.DefaultBranch, err) + ctx.JSON(http.StatusInternalServerError, private.HookPostReceiveResult{ + Err: fmt.Sprintf( + "Failed to get active PR in: %-v Branch: %s to: %-v Branch: %s Error: %v", repo, branch, baseRepo, baseRepo.DefaultBranch, err), + RepoWasEmpty: wasEmpty, }) return } - baseRepo = repo.BaseRepo - } - if !repo.IsFork && branch == baseRepo.DefaultBranch { - ctx.JSON(http.StatusOK, map[string]interface{}{ - "message": false, - }) - return + if pr == nil { + if repo.IsFork { + branch = fmt.Sprintf("%s:%s", repo.OwnerName, branch) + } + results = append(results, private.HookPostReceiveBranchResult{ + Message: true, + Create: true, + Branch: branch, + URL: fmt.Sprintf("%s/compare/%s...%s", baseRepo.HTMLURL(), util.PathEscapeSegments(baseRepo.DefaultBranch), util.PathEscapeSegments(branch)), + }) + } else { + results = append(results, private.HookPostReceiveBranchResult{ + Message: true, + Create: false, + Branch: branch, + URL: fmt.Sprintf("%s/pulls/%d", baseRepo.HTMLURL(), pr.Index), + }) + } } + } + ctx.JSON(http.StatusOK, private.HookPostReceiveResult{ + Results: results, + RepoWasEmpty: wasEmpty, + }) +} - pr, err := models.GetUnmergedPullRequest(repo.ID, baseRepo.ID, branch, baseRepo.DefaultBranch) - if err != nil && !models.IsErrPullRequestNotExist(err) { - log.Error("Failed to get active PR in: %-v Branch: %s to: %-v Branch: %s Error: %v", repo, branch, baseRepo, baseRepo.DefaultBranch, err) +// SetDefaultBranch updates the default branch +func SetDefaultBranch(ctx *macaron.Context) { + ownerName := ctx.Params(":owner") + repoName := ctx.Params(":repo") + branch := ctx.Params(":branch") + repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName) + if err != nil { + log.Error("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "Err": fmt.Sprintf("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err), + }) + return + } + if repo.OwnerName == "" { + repo.OwnerName = ownerName + } + + repo.DefaultBranch = branch + gitRepo, err := git.OpenRepository(repo.RepoPath()) + if err != nil { + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "Err": fmt.Sprintf("Failed to get git repository: %s/%s Error: %v", ownerName, repoName, err), + }) + return + } + if err := gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil { + if !git.IsErrUnsupportedVersion(err) { + gitRepo.Close() ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": fmt.Sprintf( - "Failed to get active PR in: %-v Branch: %s to: %-v Branch: %s Error: %v", repo, branch, baseRepo, baseRepo.DefaultBranch, err), + "Err": fmt.Sprintf("Unable to set default branch onrepository: %s/%s Error: %v", ownerName, repoName, err), }) return } + } + gitRepo.Close() - if pr == nil { - if repo.IsFork { - branch = fmt.Sprintf("%s:%s", repo.OwnerName, branch) - } - ctx.JSON(http.StatusOK, map[string]interface{}{ - "message": true, - "create": true, - "branch": branch, - "url": fmt.Sprintf("%s/compare/%s...%s", baseRepo.HTMLURL(), util.PathEscapeSegments(baseRepo.DefaultBranch), util.PathEscapeSegments(branch)), - }) - } else { - ctx.JSON(http.StatusOK, map[string]interface{}{ - "message": true, - "create": false, - "branch": branch, - "url": fmt.Sprintf("%s/pulls/%d", baseRepo.HTMLURL(), pr.Index), - }) - } + if err := repo.UpdateDefaultBranch(); err != nil { + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "Err": fmt.Sprintf("Unable to set default branch onrepository: %s/%s Error: %v", ownerName, repoName, err), + }) return } - ctx.JSON(http.StatusOK, map[string]interface{}{ - "message": false, - }) + ctx.PlainText(200, []byte("success")) } diff --git a/routers/private/internal.go b/routers/private/internal.go index dafcd8882..913a52e40 100644 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -10,8 +10,10 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/private" "code.gitea.io/gitea/modules/setting" + "gitea.com/macaron/binding" "gitea.com/macaron/macaron" ) @@ -77,11 +79,14 @@ func CheckUnitUser(ctx *macaron.Context) { // RegisterRoutes registers all internal APIs routes to web application. // These APIs will be invoked by internal commands for example `gitea serv` and etc. func RegisterRoutes(m *macaron.Macaron) { + bind := binding.Bind + m.Group("/", func() { m.Post("/ssh/authorized_keys", AuthorizedPublicKeyByContent) m.Post("/ssh/:id/update/:repoid", UpdatePublicKeyInRepo) - m.Get("/hook/pre-receive/:owner/:repo", HookPreReceive) - m.Get("/hook/post-receive/:owner/:repo", HookPostReceive) + m.Post("/hook/pre-receive/:owner/:repo", bind(private.HookOptions{}), HookPreReceive) + m.Post("/hook/post-receive/:owner/:repo", bind(private.HookOptions{}), HookPostReceive) + m.Post("/hook/set-default-branch/:owner/:repo/:branch", SetDefaultBranch) m.Get("/serv/none/:keyid", ServNoCommand) m.Get("/serv/command/:keyid/:owner/:repo", ServCommand) }, CheckInternalToken)