Browse Source
Refactor repository check and sync functions (#9854)
Move more general repository functions out of models/repo.gofor-closed-social
Lunny Xiao
4 years ago
committed by
zeripath
zeripath
12 changed files with 313 additions and 210 deletions
Unified View
Diff Options
-
+4 -3cmd/admin.go
-
+13 -0models/context.go
-
+3 -2models/migrations/v36.go
-
+8 -193models/repo.go
-
+11 -2modules/cron/cron.go
-
+156 -0modules/repository/check.go
-
+1 -1modules/repository/fork.go
-
+104 -0modules/repository/hooks.go
-
+1 -1modules/repository/init.go
-
+3 -2modules/repository/repo.go
-
+7 -5routers/admin/admin.go
-
+2 -1services/wiki/wiki.go
+ 4
- 3
cmd/admin.go
View File
+ 13
- 0
models/context.go
View File
+ 3
- 2
models/migrations/v36.go
View File
+ 8
- 193
models/repo.go
View File
+ 11
- 2
modules/cron/cron.go
View File
+ 156
- 0
modules/repository/check.go
View File
| @ -0,0 +1,156 @@ | |||||
| // Copyright 2020 The Gitea Authors. All rights reserved. | |||||
| // Use of this source code is governed by a MIT-style | |||||
| // license that can be found in the LICENSE file. | |||||
| package repository | |||||
| import ( | |||||
| "context" | |||||
| "fmt" | |||||
| "time" | |||||
| "code.gitea.io/gitea/models" | |||||
| "code.gitea.io/gitea/modules/git" | |||||
| "code.gitea.io/gitea/modules/log" | |||||
| "code.gitea.io/gitea/modules/setting" | |||||
| "github.com/unknwon/com" | |||||
| "xorm.io/builder" | |||||
| ) | |||||
| // GitFsck calls 'git fsck' to check repository health. | |||||
| func GitFsck(ctx context.Context) error { | |||||
| log.Trace("Doing: GitFsck") | |||||
| if err := models.Iterate( | |||||
| models.DefaultDBContext(), | |||||
| new(models.Repository), | |||||
| builder.Expr("id>0 AND is_fsck_enabled=?", true), | |||||
| func(idx int, bean interface{}) error { | |||||
| select { | |||||
| case <-ctx.Done(): | |||||
| return fmt.Errorf("Aborted due to shutdown") | |||||
| default: | |||||
| } | |||||
| repo := bean.(*models.Repository) | |||||
| repoPath := repo.RepoPath() | |||||
| log.Trace("Running health check on repository %s", repoPath) | |||||
| if err := git.Fsck(repoPath, setting.Cron.RepoHealthCheck.Timeout, setting.Cron.RepoHealthCheck.Args...); err != nil { | |||||
| desc := fmt.Sprintf("Failed to health check repository (%s): %v", repoPath, err) | |||||
| log.Warn(desc) | |||||
| if err = models.CreateRepositoryNotice(desc); err != nil { | |||||
| log.Error("CreateRepositoryNotice: %v", err) | |||||
| } | |||||
| } | |||||
| return nil | |||||
| }, | |||||
| ); err != nil { | |||||
| return err | |||||
| } | |||||
| log.Trace("Finished: GitFsck") | |||||
| return nil | |||||
| } | |||||
| // GitGcRepos calls 'git gc' to remove unnecessary files and optimize the local repository | |||||
| func GitGcRepos(ctx context.Context) error { | |||||
| log.Trace("Doing: GitGcRepos") | |||||
| args := append([]string{"gc"}, setting.Git.GCArgs...) | |||||
| if err := models.Iterate( | |||||
| models.DefaultDBContext(), | |||||
| new(models.Repository), | |||||
| builder.Gt{"id": 0}, | |||||
| func(idx int, bean interface{}) error { | |||||
| select { | |||||
| case <-ctx.Done(): | |||||
| return fmt.Errorf("Aborted due to shutdown") | |||||
| default: | |||||
| } | |||||
| repo := bean.(*models.Repository) | |||||
| if err := repo.GetOwner(); err != nil { | |||||
| return err | |||||
| } | |||||
| if stdout, err := git.NewCommand(args...). | |||||
| SetDescription(fmt.Sprintf("Repository Garbage Collection: %s", repo.FullName())). | |||||
| RunInDirTimeout( | |||||
| time.Duration(setting.Git.Timeout.GC)*time.Second, | |||||
| repo.RepoPath()); err != nil { | |||||
| log.Error("Repository garbage collection failed for %v. Stdout: %s\nError: %v", repo, stdout, err) | |||||
| return fmt.Errorf("Repository garbage collection failed: Error: %v", err) | |||||
| } | |||||
| return nil | |||||
| }, | |||||
| ); err != nil { | |||||
| return err | |||||
| } | |||||
| log.Trace("Finished: GitGcRepos") | |||||
| return nil | |||||
| } | |||||
| func gatherMissingRepoRecords() ([]*models.Repository, error) { | |||||
| repos := make([]*models.Repository, 0, 10) | |||||
| if err := models.Iterate( | |||||
| models.DefaultDBContext(), | |||||
| new(models.Repository), | |||||
| builder.Gt{"id": 0}, | |||||
| func(idx int, bean interface{}) error { | |||||
| repo := bean.(*models.Repository) | |||||
| if !com.IsDir(repo.RepoPath()) { | |||||
| repos = append(repos, repo) | |||||
| } | |||||
| return nil | |||||
| }, | |||||
| ); err != nil { | |||||
| if err2 := models.CreateRepositoryNotice(fmt.Sprintf("gatherMissingRepoRecords: %v", err)); err2 != nil { | |||||
| return nil, fmt.Errorf("CreateRepositoryNotice: %v", err) | |||||
| } | |||||
| } | |||||
| return repos, nil | |||||
| } | |||||
| // DeleteMissingRepositories deletes all repository records that lost Git files. | |||||
| func DeleteMissingRepositories(doer *models.User) error { | |||||
| repos, err := gatherMissingRepoRecords() | |||||
| if err != nil { | |||||
| return fmt.Errorf("gatherMissingRepoRecords: %v", err) | |||||
| } | |||||
| if len(repos) == 0 { | |||||
| return nil | |||||
| } | |||||
| for _, repo := range repos { | |||||
| log.Trace("Deleting %d/%d...", repo.OwnerID, repo.ID) | |||||
| if err := models.DeleteRepository(doer, repo.OwnerID, repo.ID); err != nil { | |||||
| if err2 := models.CreateRepositoryNotice(fmt.Sprintf("DeleteRepository [%d]: %v", repo.ID, err)); err2 != nil { | |||||
| return fmt.Errorf("CreateRepositoryNotice: %v", err) | |||||
| } | |||||
| } | |||||
| } | |||||
| return nil | |||||
| } | |||||
| // ReinitMissingRepositories reinitializes all repository records that lost Git files. | |||||
| func ReinitMissingRepositories() error { | |||||
| repos, err := gatherMissingRepoRecords() | |||||
| if err != nil { | |||||
| return fmt.Errorf("gatherMissingRepoRecords: %v", err) | |||||
| } | |||||
| if len(repos) == 0 { | |||||
| return nil | |||||
| } | |||||
| for _, repo := range repos { | |||||
| log.Trace("Initializing %d/%d...", repo.OwnerID, repo.ID) | |||||
| if err := git.InitRepository(repo.RepoPath(), true); err != nil { | |||||
| if err2 := models.CreateRepositoryNotice(fmt.Sprintf("InitRepository [%d]: %v", repo.ID, err)); err2 != nil { | |||||
| return fmt.Errorf("CreateRepositoryNotice: %v", err) | |||||
| } | |||||
| } | |||||
| } | |||||
| return nil | |||||
| } | |||||
+ 1
- 1
modules/repository/fork.go
View File
+ 104
- 0
modules/repository/hooks.go
View File
| @ -0,0 +1,104 @@ | |||||
| // Copyright 2020 The Gitea Authors. All rights reserved. | |||||
| // Use of this source code is governed by a MIT-style | |||||
| // license that can be found in the LICENSE file. | |||||
| package repository | |||||
| import ( | |||||
| "context" | |||||
| "fmt" | |||||
| "io/ioutil" | |||||
| "os" | |||||
| "path/filepath" | |||||
| "code.gitea.io/gitea/models" | |||||
| "code.gitea.io/gitea/modules/log" | |||||
| "code.gitea.io/gitea/modules/setting" | |||||
| "xorm.io/builder" | |||||
| ) | |||||
| // CreateDelegateHooks creates all the hooks scripts for the repo | |||||
| func CreateDelegateHooks(repoPath string) error { | |||||
| return createDelegateHooks(repoPath) | |||||
| } | |||||
| // createDelegateHooks creates all the hooks scripts for the repo | |||||
| func createDelegateHooks(repoPath string) (err error) { | |||||
| var ( | |||||
| hookNames = []string{"pre-receive", "update", "post-receive"} | |||||
| hookTpls = []string{ | |||||
| fmt.Sprintf("#!/usr/bin/env %s\ndata=$(cat)\nexitcodes=\"\"\nhookname=$(basename $0)\nGIT_DIR=${GIT_DIR:-$(dirname $0)}\n\nfor hook in ${GIT_DIR}/hooks/${hookname}.d/*; do\ntest -x \"${hook}\" || continue\necho \"${data}\" | \"${hook}\"\nexitcodes=\"${exitcodes} $?\"\ndone\n\nfor i in ${exitcodes}; do\n[ ${i} -eq 0 ] || exit ${i}\ndone\n", setting.ScriptType), | |||||
| fmt.Sprintf("#!/usr/bin/env %s\nexitcodes=\"\"\nhookname=$(basename $0)\nGIT_DIR=${GIT_DIR:-$(dirname $0)}\n\nfor hook in ${GIT_DIR}/hooks/${hookname}.d/*; do\ntest -x \"${hook}\" || continue\n\"${hook}\" $1 $2 $3\nexitcodes=\"${exitcodes} $?\"\ndone\n\nfor i in ${exitcodes}; do\n[ ${i} -eq 0 ] || exit ${i}\ndone\n", setting.ScriptType), | |||||
| fmt.Sprintf("#!/usr/bin/env %s\ndata=$(cat)\nexitcodes=\"\"\nhookname=$(basename $0)\nGIT_DIR=${GIT_DIR:-$(dirname $0)}\n\nfor hook in ${GIT_DIR}/hooks/${hookname}.d/*; do\ntest -x \"${hook}\" || continue\necho \"${data}\" | \"${hook}\"\nexitcodes=\"${exitcodes} $?\"\ndone\n\nfor i in ${exitcodes}; do\n[ ${i} -eq 0 ] || exit ${i}\ndone\n", setting.ScriptType), | |||||
| } | |||||
| giteaHookTpls = []string{ | |||||
| fmt.Sprintf("#!/usr/bin/env %s\n\"%s\" hook --config='%s' pre-receive\n", setting.ScriptType, setting.AppPath, setting.CustomConf), | |||||
| fmt.Sprintf("#!/usr/bin/env %s\n\"%s\" hook --config='%s' update $1 $2 $3\n", setting.ScriptType, setting.AppPath, setting.CustomConf), | |||||
| fmt.Sprintf("#!/usr/bin/env %s\n\"%s\" hook --config='%s' post-receive\n", setting.ScriptType, setting.AppPath, setting.CustomConf), | |||||
| } | |||||
| ) | |||||
| hookDir := filepath.Join(repoPath, "hooks") | |||||
| for i, hookName := range hookNames { | |||||
| oldHookPath := filepath.Join(hookDir, hookName) | |||||
| newHookPath := filepath.Join(hookDir, hookName+".d", "gitea") | |||||
| if err := os.MkdirAll(filepath.Join(hookDir, hookName+".d"), os.ModePerm); err != nil { | |||||
| return fmt.Errorf("create hooks dir '%s': %v", filepath.Join(hookDir, hookName+".d"), err) | |||||
| } | |||||
| // WARNING: This will override all old server-side hooks | |||||
| if err = os.Remove(oldHookPath); err != nil && !os.IsNotExist(err) { | |||||
| return fmt.Errorf("unable to pre-remove old hook file '%s' prior to rewriting: %v ", oldHookPath, err) | |||||
| } | |||||
| if err = ioutil.WriteFile(oldHookPath, []byte(hookTpls[i]), 0777); err != nil { | |||||
| return fmt.Errorf("write old hook file '%s': %v", oldHookPath, err) | |||||
| } | |||||
| if err = os.Remove(newHookPath); err != nil && !os.IsNotExist(err) { | |||||
| return fmt.Errorf("unable to pre-remove new hook file '%s' prior to rewriting: %v", newHookPath, err) | |||||
| } | |||||
| if err = ioutil.WriteFile(newHookPath, []byte(giteaHookTpls[i]), 0777); err != nil { | |||||
| return fmt.Errorf("write new hook file '%s': %v", newHookPath, err) | |||||
| } | |||||
| } | |||||
| return nil | |||||
| } | |||||
| // SyncRepositoryHooks rewrites all repositories' pre-receive, update and post-receive hooks | |||||
| // to make sure the binary and custom conf path are up-to-date. | |||||
| func SyncRepositoryHooks(ctx context.Context) error { | |||||
| log.Trace("Doing: SyncRepositoryHooks") | |||||
| if err := models.Iterate( | |||||
| models.DefaultDBContext(), | |||||
| new(models.Repository), | |||||
| builder.Gt{"id": 0}, | |||||
| func(idx int, bean interface{}) error { | |||||
| select { | |||||
| case <-ctx.Done(): | |||||
| return fmt.Errorf("Aborted due to shutdown") | |||||
| default: | |||||
| } | |||||
| if err := createDelegateHooks(bean.(*models.Repository).RepoPath()); err != nil { | |||||
| return fmt.Errorf("SyncRepositoryHook: %v", err) | |||||
| } | |||||
| if bean.(*models.Repository).HasWiki() { | |||||
| if err := createDelegateHooks(bean.(*models.Repository).WikiPath()); err != nil { | |||||
| return fmt.Errorf("SyncRepositoryHook: %v", err) | |||||
| } | |||||
| } | |||||
| return nil | |||||
| }, | |||||
| ); err != nil { | |||||
| return err | |||||
| } | |||||
| log.Trace("Finished: SyncRepositoryHooks") | |||||
| return nil | |||||
| } | |||||