Browse Source
Move migrating repository from frontend to backend (#6200)
* move migrating to backend * add loading image when migrating and fix tests * fix format * fix lint * add redis task queue support and improve docs * add redis vendor * fix vet * add database migrations and fix app.ini sample * add comments for task section on app.ini.sample * Update models/migrations/v84.go Co-Authored-By: lunny <xiaolunwen@gmail.com> * Update models/repo.go Co-Authored-By: lunny <xiaolunwen@gmail.com> * move migrating to backend * add loading image when migrating and fix tests * fix fmt * add redis task queue support and improve docs * fix fixtures * fix fixtures * fix duplicate function on index.js * fix tests * rename repository statuses * check if repository is being create when SSH request * fix lint * fix template * some improvements * fix template * unified migrate options * fix lint * fix loading page * refactor * When gitea restart, don't restart the running tasks because we may have servel gitea instances, that may break the migration * fix js * Update models/repo.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update docs/content/doc/advanced/config-cheat-sheet.en-us.md Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * fix tests * rename ErrTaskIsNotExist to ErrTaskDoesNotExist * delete release after add one on tests to make it run happy * fix tests * fix tests * improve codes * fix lint * fix lint * fix migrationsfor-closed-social
Lunny Xiao
5 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
37 changed files with 1180 additions and 210 deletions
Unified View
Diff Options
-
+1 -0.gitignore
-
+9 -0custom/conf/app.ini.sample
-
+7 -0docs/content/doc/advanced/config-cheat-sheet.en-us.md
-
+7 -0docs/content/doc/advanced/config-cheat-sheet.zh-cn.md
-
+42 -1models/fixtures/repository.yml
-
+2 -0models/migrations/migrations.go
-
+34 -0models/migrations/v99.go
-
+1 -0models/models.go
-
+51 -32models/repo.go
-
+240 -0models/task.go
-
+20 -15modules/context/repo.go
-
+3 -18modules/migrations/base/options.go
-
+27 -9modules/migrations/gitea.go
-
+4 -3modules/migrations/gitea_test.go
-
+2 -2modules/migrations/github.go
-
+9 -3modules/migrations/migrate.go
-
+1 -0modules/setting/setting.go
-
+25 -0modules/setting/task.go
-
+12 -4modules/structs/repo.go
-
+34 -0modules/structs/task.go
-
+120 -0modules/task/migrate.go
-
+14 -0modules/task/queue.go
-
+48 -0modules/task/queue_channel.go
-
+130 -0modules/task/queue_redis.go
-
+66 -0modules/task/task.go
-
+2 -0options/locale/locale_en-US.ini
-
BIN
-
+36 -0public/js/index.js
-
+2 -2routers/api/v1/repo/repo.go
-
+4 -0routers/init.go
-
+9 -0routers/private/serv.go
-
+62 -41routers/repo/repo.go
-
+30 -0routers/repo/view.go
-
+2 -0routers/routes/routes.go
-
+20 -9services/mirror/mirror_test.go
-
+73 -71templates/repo/header.tmpl
-
+31 -0templates/repo/migrating.tmpl
+ 1
- 0
.gitignore
View File
+ 9
- 0
custom/conf/app.ini.sample
View File
+ 7
- 0
docs/content/doc/advanced/config-cheat-sheet.en-us.md
View File
+ 7
- 0
docs/content/doc/advanced/config-cheat-sheet.zh-cn.md
View File
+ 42
- 1
models/fixtures/repository.yml
View File
+ 2
- 0
models/migrations/migrations.go
View File
+ 34
- 0
models/migrations/v99.go
View File
| @ -0,0 +1,34 @@ | |||||
| // Copyright 2019 The Gitea Authors. All rights reserved. | |||||
| // Use of this source code is governed by a MIT-style | |||||
| // license that can be found in the LICENSE file. | |||||
| package migrations | |||||
| import ( | |||||
| "code.gitea.io/gitea/modules/structs" | |||||
| "code.gitea.io/gitea/modules/timeutil" | |||||
| "github.com/go-xorm/xorm" | |||||
| ) | |||||
| func addTaskTable(x *xorm.Engine) error { | |||||
| type Task struct { | |||||
| ID int64 | |||||
| DoerID int64 `xorm:"index"` // operator | |||||
| OwnerID int64 `xorm:"index"` // repo owner id, when creating, the repoID maybe zero | |||||
| RepoID int64 `xorm:"index"` | |||||
| Type structs.TaskType | |||||
| Status structs.TaskStatus `xorm:"index"` | |||||
| StartTime timeutil.TimeStamp | |||||
| EndTime timeutil.TimeStamp | |||||
| PayloadContent string `xorm:"TEXT"` | |||||
| Errors string `xorm:"TEXT"` // if task failed, saved the error reason | |||||
| Created timeutil.TimeStamp `xorm:"created"` | |||||
| } | |||||
| type Repository struct { | |||||
| Status int `xorm:"NOT NULL DEFAULT 0"` | |||||
| } | |||||
| return x.Sync2(new(Task), new(Repository)) | |||||
| } | |||||
+ 1
- 0
models/models.go
View File
+ 51
- 32
models/repo.go
View File
+ 240
- 0
models/task.go
View File
| @ -0,0 +1,240 @@ | |||||
| // Copyright 2019 Gitea. All rights reserved. | |||||
| // Use of this source code is governed by a MIT-style | |||||
| // license that can be found in the LICENSE file. | |||||
| package models | |||||
| import ( | |||||
| "encoding/json" | |||||
| "fmt" | |||||
| "code.gitea.io/gitea/modules/log" | |||||
| "code.gitea.io/gitea/modules/migrations/base" | |||||
| "code.gitea.io/gitea/modules/structs" | |||||
| "code.gitea.io/gitea/modules/timeutil" | |||||
| "xorm.io/builder" | |||||
| ) | |||||
| // Task represents a task | |||||
| type Task struct { | |||||
| ID int64 | |||||
| DoerID int64 `xorm:"index"` // operator | |||||
| Doer *User `xorm:"-"` | |||||
| OwnerID int64 `xorm:"index"` // repo owner id, when creating, the repoID maybe zero | |||||
| Owner *User `xorm:"-"` | |||||
| RepoID int64 `xorm:"index"` | |||||
| Repo *Repository `xorm:"-"` | |||||
| Type structs.TaskType | |||||
| Status structs.TaskStatus `xorm:"index"` | |||||
| StartTime timeutil.TimeStamp | |||||
| EndTime timeutil.TimeStamp | |||||
| PayloadContent string `xorm:"TEXT"` | |||||
| Errors string `xorm:"TEXT"` // if task failed, saved the error reason | |||||
| Created timeutil.TimeStamp `xorm:"created"` | |||||
| } | |||||
| // LoadRepo loads repository of the task | |||||
| func (task *Task) LoadRepo() error { | |||||
| return task.loadRepo(x) | |||||
| } | |||||
| func (task *Task) loadRepo(e Engine) error { | |||||
| if task.Repo != nil { | |||||
| return nil | |||||
| } | |||||
| var repo Repository | |||||
| has, err := e.ID(task.RepoID).Get(&repo) | |||||
| if err != nil { | |||||
| return err | |||||
| } else if !has { | |||||
| return ErrRepoNotExist{ | |||||
| ID: task.RepoID, | |||||
| } | |||||
| } | |||||
| task.Repo = &repo | |||||
| return nil | |||||
| } | |||||
| // LoadDoer loads do user | |||||
| func (task *Task) LoadDoer() error { | |||||
| if task.Doer != nil { | |||||
| return nil | |||||
| } | |||||
| var doer User | |||||
| has, err := x.ID(task.DoerID).Get(&doer) | |||||
| if err != nil { | |||||
| return err | |||||
| } else if !has { | |||||
| return ErrUserNotExist{ | |||||
| UID: task.DoerID, | |||||
| } | |||||
| } | |||||
| task.Doer = &doer | |||||
| return nil | |||||
| } | |||||
| // LoadOwner loads owner user | |||||
| func (task *Task) LoadOwner() error { | |||||
| if task.Owner != nil { | |||||
| return nil | |||||
| } | |||||
| var owner User | |||||
| has, err := x.ID(task.OwnerID).Get(&owner) | |||||
| if err != nil { | |||||
| return err | |||||
| } else if !has { | |||||
| return ErrUserNotExist{ | |||||
| UID: task.OwnerID, | |||||
| } | |||||
| } | |||||
| task.Owner = &owner | |||||
| return nil | |||||
| } | |||||
| // UpdateCols updates some columns | |||||
| func (task *Task) UpdateCols(cols ...string) error { | |||||
| _, err := x.ID(task.ID).Cols(cols...).Update(task) | |||||
| return err | |||||
| } | |||||
| // MigrateConfig returns task config when migrate repository | |||||
| func (task *Task) MigrateConfig() (*structs.MigrateRepoOption, error) { | |||||
| if task.Type == structs.TaskTypeMigrateRepo { | |||||
| var opts structs.MigrateRepoOption | |||||
| err := json.Unmarshal([]byte(task.PayloadContent), &opts) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } | |||||
| return &opts, nil | |||||
| } | |||||
| return nil, fmt.Errorf("Task type is %s, not Migrate Repo", task.Type.Name()) | |||||
| } | |||||
| // ErrTaskDoesNotExist represents a "TaskDoesNotExist" kind of error. | |||||
| type ErrTaskDoesNotExist struct { | |||||
| ID int64 | |||||
| RepoID int64 | |||||
| Type structs.TaskType | |||||
| } | |||||
| // IsErrTaskDoesNotExist checks if an error is a ErrTaskIsNotExist. | |||||
| func IsErrTaskDoesNotExist(err error) bool { | |||||
| _, ok := err.(ErrTaskDoesNotExist) | |||||
| return ok | |||||
| } | |||||
| func (err ErrTaskDoesNotExist) Error() string { | |||||
| return fmt.Sprintf("task is not exist [id: %d, repo_id: %d, type: %d]", | |||||
| err.ID, err.RepoID, err.Type) | |||||
| } | |||||
| // GetMigratingTask returns the migrating task by repo's id | |||||
| func GetMigratingTask(repoID int64) (*Task, error) { | |||||
| var task = Task{ | |||||
| RepoID: repoID, | |||||
| Type: structs.TaskTypeMigrateRepo, | |||||
| } | |||||
| has, err := x.Get(&task) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } else if !has { | |||||
| return nil, ErrTaskDoesNotExist{0, repoID, task.Type} | |||||
| } | |||||
| return &task, nil | |||||
| } | |||||
| // FindTaskOptions find all tasks | |||||
| type FindTaskOptions struct { | |||||
| Status int | |||||
| } | |||||
| // ToConds generates conditions for database operation. | |||||
| func (opts FindTaskOptions) ToConds() builder.Cond { | |||||
| var cond = builder.NewCond() | |||||
| if opts.Status >= 0 { | |||||
| cond = cond.And(builder.Eq{"status": opts.Status}) | |||||
| } | |||||
| return cond | |||||
| } | |||||
| // FindTasks find all tasks | |||||
| func FindTasks(opts FindTaskOptions) ([]*Task, error) { | |||||
| var tasks = make([]*Task, 0, 10) | |||||
| err := x.Where(opts.ToConds()).Find(&tasks) | |||||
| return tasks, err | |||||
| } | |||||
| func createTask(e Engine, task *Task) error { | |||||
| _, err := e.Insert(task) | |||||
| return err | |||||
| } | |||||
| // CreateMigrateTask creates a migrate task | |||||
| func CreateMigrateTask(doer, u *User, opts base.MigrateOptions) (*Task, error) { | |||||
| bs, err := json.Marshal(&opts) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } | |||||
| var task = Task{ | |||||
| DoerID: doer.ID, | |||||
| OwnerID: u.ID, | |||||
| Type: structs.TaskTypeMigrateRepo, | |||||
| Status: structs.TaskStatusQueue, | |||||
| PayloadContent: string(bs), | |||||
| } | |||||
| if err := createTask(x, &task); err != nil { | |||||
| return nil, err | |||||
| } | |||||
| repo, err := CreateRepository(doer, u, CreateRepoOptions{ | |||||
| Name: opts.RepoName, | |||||
| Description: opts.Description, | |||||
| OriginalURL: opts.CloneAddr, | |||||
| IsPrivate: opts.Private, | |||||
| IsMirror: opts.Mirror, | |||||
| Status: RepositoryBeingMigrated, | |||||
| }) | |||||
| if err != nil { | |||||
| task.EndTime = timeutil.TimeStampNow() | |||||
| task.Status = structs.TaskStatusFailed | |||||
| err2 := task.UpdateCols("end_time", "status") | |||||
| if err2 != nil { | |||||
| log.Error("UpdateCols Failed: %v", err2.Error()) | |||||
| } | |||||
| return nil, err | |||||
| } | |||||
| task.RepoID = repo.ID | |||||
| if err = task.UpdateCols("repo_id"); err != nil { | |||||
| return nil, err | |||||
| } | |||||
| return &task, nil | |||||
| } | |||||
| // FinishMigrateTask updates database when migrate task finished | |||||
| func FinishMigrateTask(task *Task) error { | |||||
| task.Status = structs.TaskStatusFinished | |||||
| task.EndTime = timeutil.TimeStampNow() | |||||
| sess := x.NewSession() | |||||
| defer sess.Close() | |||||
| if err := sess.Begin(); err != nil { | |||||
| return err | |||||
| } | |||||
| if _, err := sess.ID(task.ID).Cols("status", "end_time").Update(task); err != nil { | |||||
| return err | |||||
| } | |||||
| task.Repo.Status = RepositoryReady | |||||
| if _, err := sess.ID(task.RepoID).Cols("status").Update(task.Repo); err != nil { | |||||
| return err | |||||
| } | |||||
| return sess.Commit() | |||||
| } | |||||
+ 20
- 15
modules/context/repo.go
View File
+ 3
- 18
modules/migrations/base/options.go
View File
+ 27
- 9
modules/migrations/gitea.go
View File
+ 4
- 3
modules/migrations/gitea_test.go
View File
+ 2
- 2
modules/migrations/github.go
View File
+ 9
- 3
modules/migrations/migrate.go
View File
+ 1
- 0
modules/setting/setting.go
View File
+ 25
- 0
modules/setting/task.go
View File
| @ -0,0 +1,25 @@ | |||||
| // Copyright 2019 The Gitea Authors. All rights reserved. | |||||
| // Use of this source code is governed by a MIT-style | |||||
| // license that can be found in the LICENSE file. | |||||
| package setting | |||||
| var ( | |||||
| // Task settings | |||||
| Task = struct { | |||||
| QueueType string | |||||
| QueueLength int | |||||
| QueueConnStr string | |||||
| }{ | |||||
| QueueType: ChannelQueueType, | |||||
| QueueLength: 1000, | |||||
| QueueConnStr: "addrs=127.0.0.1:6379 db=0", | |||||
| } | |||||
| ) | |||||
| func newTaskService() { | |||||
| sec := Cfg.Section("task") | |||||
| Task.QueueType = sec.Key("QUEUE_TYPE").MustString(ChannelQueueType) | |||||
| Task.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000) | |||||
| Task.QueueConnStr = sec.Key("QUEUE_CONN_STR").MustString("addrs=127.0.0.1:6379 db=0") | |||||
| } | |||||
+ 12
- 4
modules/structs/repo.go
View File
+ 34
- 0
modules/structs/task.go
View File
| @ -0,0 +1,34 @@ | |||||
| // Copyright 2019 Gitea. All rights reserved. | |||||
| // Use of this source code is governed by a MIT-style | |||||
| // license that can be found in the LICENSE file. | |||||
| package structs | |||||
| // TaskType defines task type | |||||
| type TaskType int | |||||
| // all kinds of task types | |||||
| const ( | |||||
| TaskTypeMigrateRepo TaskType = iota // migrate repository from external or local disk | |||||
| ) | |||||
| // Name returns the task type name | |||||
| func (taskType TaskType) Name() string { | |||||
| switch taskType { | |||||
| case TaskTypeMigrateRepo: | |||||
| return "Migrate Repository" | |||||
| } | |||||
| return "" | |||||
| } | |||||
| // TaskStatus defines task status | |||||
| type TaskStatus int | |||||
| // enumerate all the kinds of task status | |||||
| const ( | |||||
| TaskStatusQueue TaskStatus = iota // 0 task is queue | |||||
| TaskStatusRunning // 1 task is running | |||||
| TaskStatusStopped // 2 task is stopped | |||||
| TaskStatusFailed // 3 task is failed | |||||
| TaskStatusFinished // 4 task is finished | |||||
| ) | |||||
+ 120
- 0
modules/task/migrate.go
View File
| @ -0,0 +1,120 @@ | |||||
| // Copyright 2019 Gitea. All rights reserved. | |||||
| // Use of this source code is governed by a MIT-style | |||||
| // license that can be found in the LICENSE file. | |||||
| package task | |||||
| import ( | |||||
| "bytes" | |||||
| "errors" | |||||
| "fmt" | |||||
| "strings" | |||||
| "code.gitea.io/gitea/models" | |||||
| "code.gitea.io/gitea/modules/log" | |||||
| "code.gitea.io/gitea/modules/migrations" | |||||
| "code.gitea.io/gitea/modules/notification" | |||||
| "code.gitea.io/gitea/modules/structs" | |||||
| "code.gitea.io/gitea/modules/timeutil" | |||||
| "code.gitea.io/gitea/modules/util" | |||||
| ) | |||||
| func handleCreateError(owner *models.User, err error, name string) error { | |||||
| switch { | |||||
| case models.IsErrReachLimitOfRepo(err): | |||||
| return fmt.Errorf("You have already reached your limit of %d repositories", owner.MaxCreationLimit()) | |||||
| case models.IsErrRepoAlreadyExist(err): | |||||
| return errors.New("The repository name is already used") | |||||
| case models.IsErrNameReserved(err): | |||||
| return fmt.Errorf("The repository name '%s' is reserved", err.(models.ErrNameReserved).Name) | |||||
| case models.IsErrNamePatternNotAllowed(err): | |||||
| return fmt.Errorf("The pattern '%s' is not allowed in a repository name", err.(models.ErrNamePatternNotAllowed).Pattern) | |||||
| default: | |||||
| return err | |||||
| } | |||||
| } | |||||
| func runMigrateTask(t *models.Task) (err error) { | |||||
| defer func() { | |||||
| if e := recover(); e != nil { | |||||
| var buf bytes.Buffer | |||||
| fmt.Fprintf(&buf, "Handler crashed with error: %v", log.Stack(2)) | |||||
| err = errors.New(buf.String()) | |||||
| } | |||||
| if err == nil { | |||||
| err = models.FinishMigrateTask(t) | |||||
| if err == nil { | |||||
| notification.NotifyMigrateRepository(t.Doer, t.Owner, t.Repo) | |||||
| return | |||||
| } | |||||
| log.Error("FinishMigrateTask failed: %s", err.Error()) | |||||
| } | |||||
| t.EndTime = timeutil.TimeStampNow() | |||||
| t.Status = structs.TaskStatusFailed | |||||
| t.Errors = err.Error() | |||||
| if err := t.UpdateCols("status", "errors", "end_time"); err != nil { | |||||
| log.Error("Task UpdateCols failed: %s", err.Error()) | |||||
| } | |||||
| if t.Repo != nil { | |||||
| if errDelete := models.DeleteRepository(t.Doer, t.OwnerID, t.Repo.ID); errDelete != nil { | |||||
| log.Error("DeleteRepository: %v", errDelete) | |||||
| } | |||||
| } | |||||
| }() | |||||
| if err := t.LoadRepo(); err != nil { | |||||
| return err | |||||
| } | |||||
| // if repository is ready, then just finsih the task | |||||
| if t.Repo.Status == models.RepositoryReady { | |||||
| return nil | |||||
| } | |||||
| if err := t.LoadDoer(); err != nil { | |||||
| return err | |||||
| } | |||||
| if err := t.LoadOwner(); err != nil { | |||||
| return err | |||||
| } | |||||
| t.StartTime = timeutil.TimeStampNow() | |||||
| t.Status = structs.TaskStatusRunning | |||||
| if err := t.UpdateCols("start_time", "status"); err != nil { | |||||
| return err | |||||
| } | |||||
| var opts *structs.MigrateRepoOption | |||||
| opts, err = t.MigrateConfig() | |||||
| if err != nil { | |||||
| return err | |||||
| } | |||||
| opts.MigrateToRepoID = t.RepoID | |||||
| repo, err := migrations.MigrateRepository(t.Doer, t.Owner.Name, *opts) | |||||
| if err == nil { | |||||
| notification.NotifyMigrateRepository(t.Doer, t.Owner, repo) | |||||
| log.Trace("Repository migrated [%d]: %s/%s", repo.ID, t.Owner.Name, repo.Name) | |||||
| return nil | |||||
| } | |||||
| if models.IsErrRepoAlreadyExist(err) { | |||||
| return errors.New("The repository name is already used") | |||||
| } | |||||
| // remoteAddr may contain credentials, so we sanitize it | |||||
| err = util.URLSanitizedError(err, opts.CloneAddr) | |||||
| if strings.Contains(err.Error(), "Authentication failed") || | |||||
| strings.Contains(err.Error(), "could not read Username") { | |||||
| return fmt.Errorf("Authentication failed: %v", err.Error()) | |||||
| } else if strings.Contains(err.Error(), "fatal:") { | |||||
| return fmt.Errorf("Migration failed: %v", err.Error()) | |||||
| } | |||||
| return handleCreateError(t.Owner, err, "MigratePost") | |||||
| } | |||||
+ 14
- 0
modules/task/queue.go
View File
| @ -0,0 +1,14 @@ | |||||
| // Copyright 2019 Gitea. All rights reserved. | |||||
| // Use of this source code is governed by a MIT-style | |||||
| // license that can be found in the LICENSE file. | |||||
| package task | |||||
| import "code.gitea.io/gitea/models" | |||||
| // Queue defines an interface to run task queue | |||||
| type Queue interface { | |||||
| Run() error | |||||
| Push(*models.Task) error | |||||
| Stop() | |||||
| } | |||||
+ 48
- 0
modules/task/queue_channel.go
View File
| @ -0,0 +1,48 @@ | |||||
| // Copyright 2019 The Gitea Authors. All rights reserved. | |||||
| // Use of this source code is governed by a MIT-style | |||||
| // license that can be found in the LICENSE file. | |||||
| package task | |||||
| import ( | |||||
| "code.gitea.io/gitea/models" | |||||
| "code.gitea.io/gitea/modules/log" | |||||
| ) | |||||
| var ( | |||||
| _ Queue = &ChannelQueue{} | |||||
| ) | |||||
| // ChannelQueue implements | |||||
| type ChannelQueue struct { | |||||
| queue chan *models.Task | |||||
| } | |||||
| // NewChannelQueue create a memory channel queue | |||||
| func NewChannelQueue(queueLen int) *ChannelQueue { | |||||
| return &ChannelQueue{ | |||||
| queue: make(chan *models.Task, queueLen), | |||||
| } | |||||
| } | |||||
| // Run starts to run the queue | |||||
| func (c *ChannelQueue) Run() error { | |||||
| for task := range c.queue { | |||||
| err := Run(task) | |||||
| if err != nil { | |||||
| log.Error("Run task failed: %s", err.Error()) | |||||
| } | |||||
| } | |||||
| return nil | |||||
| } | |||||
| // Push will push the task ID to queue | |||||
| func (c *ChannelQueue) Push(task *models.Task) error { | |||||
| c.queue <- task | |||||
| return nil | |||||
| } | |||||
| // Stop stop the queue | |||||
| func (c *ChannelQueue) Stop() { | |||||
| close(c.queue) | |||||
| } | |||||
+ 130
- 0
modules/task/queue_redis.go
View File
| @ -0,0 +1,130 @@ | |||||
| // Copyright 2019 The Gitea Authors. All rights reserved. | |||||
| // Use of this source code is governed by a MIT-style | |||||
| // license that can be found in the LICENSE file. | |||||
| package task | |||||
| import ( | |||||
| "encoding/json" | |||||
| "errors" | |||||
| "strconv" | |||||
| "strings" | |||||
| "time" | |||||
| "code.gitea.io/gitea/models" | |||||
| "code.gitea.io/gitea/modules/log" | |||||
| "github.com/go-redis/redis" | |||||
| ) | |||||
| var ( | |||||
| _ Queue = &RedisQueue{} | |||||
| ) | |||||
| type redisClient interface { | |||||
| RPush(key string, args ...interface{}) *redis.IntCmd | |||||
| LPop(key string) *redis.StringCmd | |||||
| Ping() *redis.StatusCmd | |||||
| } | |||||
| // RedisQueue redis queue | |||||
| type RedisQueue struct { | |||||
| client redisClient | |||||
| queueName string | |||||
| closeChan chan bool | |||||
| } | |||||
| func parseConnStr(connStr string) (addrs, password string, dbIdx int, err error) { | |||||
| fields := strings.Fields(connStr) | |||||
| for _, f := range fields { | |||||
| items := strings.SplitN(f, "=", 2) | |||||
| if len(items) < 2 { | |||||
| continue | |||||
| } | |||||
| switch strings.ToLower(items[0]) { | |||||
| case "addrs": | |||||
| addrs = items[1] | |||||
| case "password": | |||||
| password = items[1] | |||||
| case "db": | |||||
| dbIdx, err = strconv.Atoi(items[1]) | |||||
| if err != nil { | |||||
| return | |||||
| } | |||||
| } | |||||
| } | |||||
| return | |||||
| } | |||||
| // NewRedisQueue creates single redis or cluster redis queue | |||||
| func NewRedisQueue(addrs string, password string, dbIdx int) (*RedisQueue, error) { | |||||
| dbs := strings.Split(addrs, ",") | |||||
| var queue = RedisQueue{ | |||||
| queueName: "task_queue", | |||||
| closeChan: make(chan bool), | |||||
| } | |||||
| if len(dbs) == 0 { | |||||
| return nil, errors.New("no redis host found") | |||||
| } else if len(dbs) == 1 { | |||||
| queue.client = redis.NewClient(&redis.Options{ | |||||
| Addr: strings.TrimSpace(dbs[0]), // use default Addr | |||||
| Password: password, // no password set | |||||
| DB: dbIdx, // use default DB | |||||
| }) | |||||
| } else { | |||||
| // cluster will ignore db | |||||
| queue.client = redis.NewClusterClient(&redis.ClusterOptions{ | |||||
| Addrs: dbs, | |||||
| Password: password, | |||||
| }) | |||||
| } | |||||
| if err := queue.client.Ping().Err(); err != nil { | |||||
| return nil, err | |||||
| } | |||||
| return &queue, nil | |||||
| } | |||||
| // Run starts to run the queue | |||||
| func (r *RedisQueue) Run() error { | |||||
| for { | |||||
| select { | |||||
| case <-r.closeChan: | |||||
| return nil | |||||
| case <-time.After(time.Millisecond * 100): | |||||
| } | |||||
| bs, err := r.client.LPop(r.queueName).Bytes() | |||||
| if err != nil { | |||||
| if err != redis.Nil { | |||||
| log.Error("LPop failed: %v", err) | |||||
| } | |||||
| time.Sleep(time.Millisecond * 100) | |||||
| continue | |||||
| } | |||||
| var task models.Task | |||||
| err = json.Unmarshal(bs, &task) | |||||
| if err != nil { | |||||
| log.Error("Unmarshal task failed: %s", err.Error()) | |||||
| } else { | |||||
| err = Run(&task) | |||||
| if err != nil { | |||||
| log.Error("Run task failed: %s", err.Error()) | |||||
| } | |||||
| } | |||||
| } | |||||
| } | |||||
| // Push implements Queue | |||||
| func (r *RedisQueue) Push(task *models.Task) error { | |||||
| bs, err := json.Marshal(task) | |||||
| if err != nil { | |||||
| return err | |||||
| } | |||||
| return r.client.RPush(r.queueName, bs).Err() | |||||
| } | |||||
| // Stop stop the queue | |||||
| func (r *RedisQueue) Stop() { | |||||
| r.closeChan <- true | |||||
| } | |||||
+ 66
- 0
modules/task/task.go
View File
| @ -0,0 +1,66 @@ | |||||
| // Copyright 2019 Gitea. All rights reserved. | |||||
| // Use of this source code is governed by a MIT-style | |||||
| // license that can be found in the LICENSE file. | |||||
| package task | |||||
| import ( | |||||
| "fmt" | |||||
| "code.gitea.io/gitea/models" | |||||
| "code.gitea.io/gitea/modules/log" | |||||
| "code.gitea.io/gitea/modules/migrations/base" | |||||
| "code.gitea.io/gitea/modules/setting" | |||||
| "code.gitea.io/gitea/modules/structs" | |||||
| ) | |||||
| // taskQueue is a global queue of tasks | |||||
| var taskQueue Queue | |||||
| // Run a task | |||||
| func Run(t *models.Task) error { | |||||
| switch t.Type { | |||||
| case structs.TaskTypeMigrateRepo: | |||||
| return runMigrateTask(t) | |||||
| default: | |||||
| return fmt.Errorf("Unknow task type: %d", t.Type) | |||||
| } | |||||
| } | |||||
| // Init will start the service to get all unfinished tasks and run them | |||||
| func Init() error { | |||||
| switch setting.Task.QueueType { | |||||
| case setting.ChannelQueueType: | |||||
| taskQueue = NewChannelQueue(setting.Task.QueueLength) | |||||
| case setting.RedisQueueType: | |||||
| var err error | |||||
| addrs, pass, idx, err := parseConnStr(setting.Task.QueueConnStr) | |||||
| if err != nil { | |||||
| return err | |||||
| } | |||||
| taskQueue, err = NewRedisQueue(addrs, pass, idx) | |||||
| if err != nil { | |||||
| return err | |||||
| } | |||||
| default: | |||||
| return fmt.Errorf("Unsupported task queue type: %v", setting.Task.QueueType) | |||||
| } | |||||
| go func() { | |||||
| if err := taskQueue.Run(); err != nil { | |||||
| log.Error("taskQueue.Run end failed: %v", err) | |||||
| } | |||||
| }() | |||||
| return nil | |||||
| } | |||||
| // MigrateRepository add migration repository to task | |||||
| func MigrateRepository(doer, u *models.User, opts base.MigrateOptions) error { | |||||
| task, err := models.CreateMigrateTask(doer, u, opts) | |||||
| if err != nil { | |||||
| return err | |||||
| } | |||||
| return taskQueue.Push(task) | |||||
| } | |||||
+ 2
- 0
options/locale/locale_en-US.ini
View File
BIN
View File
+ 36
- 0
public/js/index.js
View File
+ 2
- 2
routers/api/v1/repo/repo.go
View File
+ 4
- 0
routers/init.go
View File
+ 9
- 0
routers/private/serv.go
View File
+ 62
- 41
routers/repo/repo.go
View File
+ 30
- 0
routers/repo/view.go
View File
+ 2
- 0
routers/routes/routes.go
View File
+ 20
- 9
services/mirror/mirror_test.go
View File
+ 73
- 71
templates/repo/header.tmpl
View File
+ 31
- 0
templates/repo/migrating.tmpl
View File
| @ -0,0 +1,31 @@ | |||||
| {{template "base/head" .}} | |||||
| <div class="repository quickstart"> | |||||
| {{template "repo/header" .}} | |||||
| <div class="ui container"> | |||||
| <div class="ui grid"> | |||||
| <div class="sixteen wide column content"> | |||||
| {{template "base/alert" .}} | |||||
| <div class="home"> | |||||
| <div class="ui stackable middle very relaxed page grid"> | |||||
| <div id="repo_migrating" class="sixteen wide center aligned centered column" repo="{{.Repo.Repository.FullName}}"> | |||||
| <div> | |||||
| <img src="{{AppSubUrl}}/img/loading.png"/> | |||||
| </div> | |||||
| </div> | |||||
| </div> | |||||
| <div class="ui stackable middle very relaxed page grid"> | |||||
| <div class="sixteen wide center aligned centered column"> | |||||
| <div id="repo_migrating_progress"> | |||||
| <p>{{.i18n.Tr "repo.migrate.migrating" .CloneAddr | Safe}}</p> | |||||
| </div> | |||||
| <div id="repo_migrating_failed"> | |||||
| <p>{{.i18n.Tr "repo.migrate.migrating_failed" .CloneAddr | Safe}}</p> | |||||
| </div> | |||||
| </div> | |||||
| </div> | |||||
| </div> | |||||
| </div> | |||||
| </div> | |||||
| </div> | |||||
| </div> | |||||
| {{template "base/footer" .}} | |||||