Browse Source
Move migrating repository from frontend to backend (#6200)
* move migrating to backend * add loading image when migrating and fix tests * fix format * fix lint * add redis task queue support and improve docs * add redis vendor * fix vet * add database migrations and fix app.ini sample * add comments for task section on app.ini.sample * Update models/migrations/v84.go Co-Authored-By: lunny <xiaolunwen@gmail.com> * Update models/repo.go Co-Authored-By: lunny <xiaolunwen@gmail.com> * move migrating to backend * add loading image when migrating and fix tests * fix fmt * add redis task queue support and improve docs * fix fixtures * fix fixtures * fix duplicate function on index.js * fix tests * rename repository statuses * check if repository is being create when SSH request * fix lint * fix template * some improvements * fix template * unified migrate options * fix lint * fix loading page * refactor * When gitea restart, don't restart the running tasks because we may have servel gitea instances, that may break the migration * fix js * Update models/repo.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update docs/content/doc/advanced/config-cheat-sheet.en-us.md Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * fix tests * rename ErrTaskIsNotExist to ErrTaskDoesNotExist * delete release after add one on tests to make it run happy * fix tests * fix tests * improve codes * fix lint * fix lint * fix migrationsfor-closed-social
Lunny Xiao
5 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
37 changed files with 1180 additions and 210 deletions
Split View
Diff Options
-
+1 -0.gitignore
-
+9 -0custom/conf/app.ini.sample
-
+7 -0docs/content/doc/advanced/config-cheat-sheet.en-us.md
-
+7 -0docs/content/doc/advanced/config-cheat-sheet.zh-cn.md
-
+42 -1models/fixtures/repository.yml
-
+2 -0models/migrations/migrations.go
-
+34 -0models/migrations/v99.go
-
+1 -0models/models.go
-
+51 -32models/repo.go
-
+240 -0models/task.go
-
+20 -15modules/context/repo.go
-
+3 -18modules/migrations/base/options.go
-
+27 -9modules/migrations/gitea.go
-
+4 -3modules/migrations/gitea_test.go
-
+2 -2modules/migrations/github.go
-
+9 -3modules/migrations/migrate.go
-
+1 -0modules/setting/setting.go
-
+25 -0modules/setting/task.go
-
+12 -4modules/structs/repo.go
-
+34 -0modules/structs/task.go
-
+120 -0modules/task/migrate.go
-
+14 -0modules/task/queue.go
-
+48 -0modules/task/queue_channel.go
-
+130 -0modules/task/queue_redis.go
-
+66 -0modules/task/task.go
-
+2 -0options/locale/locale_en-US.ini
-
BIN
-
+36 -0public/js/index.js
-
+2 -2routers/api/v1/repo/repo.go
-
+4 -0routers/init.go
-
+9 -0routers/private/serv.go
-
+62 -41routers/repo/repo.go
-
+30 -0routers/repo/view.go
-
+2 -0routers/routes/routes.go
-
+20 -9services/mirror/mirror_test.go
-
+73 -71templates/repo/header.tmpl
-
+31 -0templates/repo/migrating.tmpl
+ 1
- 0
.gitignore
View File
+ 9
- 0
custom/conf/app.ini.sample
View File
+ 7
- 0
docs/content/doc/advanced/config-cheat-sheet.en-us.md
View File
+ 7
- 0
docs/content/doc/advanced/config-cheat-sheet.zh-cn.md
View File
+ 42
- 1
models/fixtures/repository.yml
View File
+ 2
- 0
models/migrations/migrations.go
View File
+ 34
- 0
models/migrations/v99.go
View File
| @ -0,0 +1,34 @@ | |||
| // Copyright 2019 The Gitea Authors. All rights reserved. | |||
| // Use of this source code is governed by a MIT-style | |||
| // license that can be found in the LICENSE file. | |||
| package migrations | |||
| import ( | |||
| "code.gitea.io/gitea/modules/structs" | |||
| "code.gitea.io/gitea/modules/timeutil" | |||
| "github.com/go-xorm/xorm" | |||
| ) | |||
| func addTaskTable(x *xorm.Engine) error { | |||
| type Task struct { | |||
| ID int64 | |||
| DoerID int64 `xorm:"index"` // operator | |||
| OwnerID int64 `xorm:"index"` // repo owner id, when creating, the repoID maybe zero | |||
| RepoID int64 `xorm:"index"` | |||
| Type structs.TaskType | |||
| Status structs.TaskStatus `xorm:"index"` | |||
| StartTime timeutil.TimeStamp | |||
| EndTime timeutil.TimeStamp | |||
| PayloadContent string `xorm:"TEXT"` | |||
| Errors string `xorm:"TEXT"` // if task failed, saved the error reason | |||
| Created timeutil.TimeStamp `xorm:"created"` | |||
| } | |||
| type Repository struct { | |||
| Status int `xorm:"NOT NULL DEFAULT 0"` | |||
| } | |||
| return x.Sync2(new(Task), new(Repository)) | |||
| } | |||
+ 1
- 0
models/models.go
View File
+ 51
- 32
models/repo.go
View File
+ 240
- 0
models/task.go
View File
| @ -0,0 +1,240 @@ | |||
| // Copyright 2019 Gitea. All rights reserved. | |||
| // Use of this source code is governed by a MIT-style | |||
| // license that can be found in the LICENSE file. | |||
| package models | |||
| import ( | |||
| "encoding/json" | |||
| "fmt" | |||
| "code.gitea.io/gitea/modules/log" | |||
| "code.gitea.io/gitea/modules/migrations/base" | |||
| "code.gitea.io/gitea/modules/structs" | |||
| "code.gitea.io/gitea/modules/timeutil" | |||
| "xorm.io/builder" | |||
| ) | |||
| // Task represents a task | |||
| type Task struct { | |||
| ID int64 | |||
| DoerID int64 `xorm:"index"` // operator | |||
| Doer *User `xorm:"-"` | |||
| OwnerID int64 `xorm:"index"` // repo owner id, when creating, the repoID maybe zero | |||
| Owner *User `xorm:"-"` | |||
| RepoID int64 `xorm:"index"` | |||
| Repo *Repository `xorm:"-"` | |||
| Type structs.TaskType | |||
| Status structs.TaskStatus `xorm:"index"` | |||
| StartTime timeutil.TimeStamp | |||
| EndTime timeutil.TimeStamp | |||
| PayloadContent string `xorm:"TEXT"` | |||
| Errors string `xorm:"TEXT"` // if task failed, saved the error reason | |||
| Created timeutil.TimeStamp `xorm:"created"` | |||
| } | |||
| // LoadRepo loads repository of the task | |||
| func (task *Task) LoadRepo() error { | |||
| return task.loadRepo(x) | |||
| } | |||
| func (task *Task) loadRepo(e Engine) error { | |||
| if task.Repo != nil { | |||
| return nil | |||
| } | |||
| var repo Repository | |||
| has, err := e.ID(task.RepoID).Get(&repo) | |||
| if err != nil { | |||
| return err | |||
| } else if !has { | |||
| return ErrRepoNotExist{ | |||
| ID: task.RepoID, | |||
| } | |||
| } | |||
| task.Repo = &repo | |||
| return nil | |||
| } | |||
| // LoadDoer loads do user | |||
| func (task *Task) LoadDoer() error { | |||
| if task.Doer != nil { | |||
| return nil | |||
| } | |||
| var doer User | |||
| has, err := x.ID(task.DoerID).Get(&doer) | |||
| if err != nil { | |||
| return err | |||
| } else if !has { | |||
| return ErrUserNotExist{ | |||
| UID: task.DoerID, | |||
| } | |||
| } | |||
| task.Doer = &doer | |||
| return nil | |||
| } | |||
| // LoadOwner loads owner user | |||
| func (task *Task) LoadOwner() error { | |||
| if task.Owner != nil { | |||
| return nil | |||
| } | |||
| var owner User | |||
| has, err := x.ID(task.OwnerID).Get(&owner) | |||
| if err != nil { | |||
| return err | |||
| } else if !has { | |||
| return ErrUserNotExist{ | |||
| UID: task.OwnerID, | |||
| } | |||
| } | |||
| task.Owner = &owner | |||
| return nil | |||
| } | |||
| // UpdateCols updates some columns | |||
| func (task *Task) UpdateCols(cols ...string) error { | |||
| _, err := x.ID(task.ID).Cols(cols...).Update(task) | |||
| return err | |||
| } | |||
| // MigrateConfig returns task config when migrate repository | |||
| func (task *Task) MigrateConfig() (*structs.MigrateRepoOption, error) { | |||
| if task.Type == structs.TaskTypeMigrateRepo { | |||
| var opts structs.MigrateRepoOption | |||
| err := json.Unmarshal([]byte(task.PayloadContent), &opts) | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| return &opts, nil | |||
| } | |||
| return nil, fmt.Errorf("Task type is %s, not Migrate Repo", task.Type.Name()) | |||
| } | |||
| // ErrTaskDoesNotExist represents a "TaskDoesNotExist" kind of error. | |||
| type ErrTaskDoesNotExist struct { | |||
| ID int64 | |||
| RepoID int64 | |||
| Type structs.TaskType | |||
| } | |||
| // IsErrTaskDoesNotExist checks if an error is a ErrTaskIsNotExist. | |||
| func IsErrTaskDoesNotExist(err error) bool { | |||
| _, ok := err.(ErrTaskDoesNotExist) | |||
| return ok | |||
| } | |||
| func (err ErrTaskDoesNotExist) Error() string { | |||
| return fmt.Sprintf("task is not exist [id: %d, repo_id: %d, type: %d]", | |||
| err.ID, err.RepoID, err.Type) | |||
| } | |||
| // GetMigratingTask returns the migrating task by repo's id | |||
| func GetMigratingTask(repoID int64) (*Task, error) { | |||
| var task = Task{ | |||
| RepoID: repoID, | |||
| Type: structs.TaskTypeMigrateRepo, | |||
| } | |||
| has, err := x.Get(&task) | |||
| if err != nil { | |||
| return nil, err | |||
| } else if !has { | |||
| return nil, ErrTaskDoesNotExist{0, repoID, task.Type} | |||
| } | |||
| return &task, nil | |||
| } | |||
| // FindTaskOptions find all tasks | |||
| type FindTaskOptions struct { | |||
| Status int | |||
| } | |||
| // ToConds generates conditions for database operation. | |||
| func (opts FindTaskOptions) ToConds() builder.Cond { | |||
| var cond = builder.NewCond() | |||
| if opts.Status >= 0 { | |||
| cond = cond.And(builder.Eq{"status": opts.Status}) | |||
| } | |||
| return cond | |||
| } | |||
| // FindTasks find all tasks | |||
| func FindTasks(opts FindTaskOptions) ([]*Task, error) { | |||
| var tasks = make([]*Task, 0, 10) | |||
| err := x.Where(opts.ToConds()).Find(&tasks) | |||
| return tasks, err | |||
| } | |||
| func createTask(e Engine, task *Task) error { | |||
| _, err := e.Insert(task) | |||
| return err | |||
| } | |||
| // CreateMigrateTask creates a migrate task | |||
| func CreateMigrateTask(doer, u *User, opts base.MigrateOptions) (*Task, error) { | |||
| bs, err := json.Marshal(&opts) | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| var task = Task{ | |||
| DoerID: doer.ID, | |||
| OwnerID: u.ID, | |||
| Type: structs.TaskTypeMigrateRepo, | |||
| Status: structs.TaskStatusQueue, | |||
| PayloadContent: string(bs), | |||
| } | |||
| if err := createTask(x, &task); err != nil { | |||
| return nil, err | |||
| } | |||
| repo, err := CreateRepository(doer, u, CreateRepoOptions{ | |||
| Name: opts.RepoName, | |||
| Description: opts.Description, | |||
| OriginalURL: opts.CloneAddr, | |||
| IsPrivate: opts.Private, | |||
| IsMirror: opts.Mirror, | |||
| Status: RepositoryBeingMigrated, | |||
| }) | |||
| if err != nil { | |||
| task.EndTime = timeutil.TimeStampNow() | |||
| task.Status = structs.TaskStatusFailed | |||
| err2 := task.UpdateCols("end_time", "status") | |||
| if err2 != nil { | |||
| log.Error("UpdateCols Failed: %v", err2.Error()) | |||
| } | |||
| return nil, err | |||
| } | |||
| task.RepoID = repo.ID | |||
| if err = task.UpdateCols("repo_id"); err != nil { | |||
| return nil, err | |||
| } | |||
| return &task, nil | |||
| } | |||
| // FinishMigrateTask updates database when migrate task finished | |||
| func FinishMigrateTask(task *Task) error { | |||
| task.Status = structs.TaskStatusFinished | |||
| task.EndTime = timeutil.TimeStampNow() | |||
| sess := x.NewSession() | |||
| defer sess.Close() | |||
| if err := sess.Begin(); err != nil { | |||
| return err | |||
| } | |||
| if _, err := sess.ID(task.ID).Cols("status", "end_time").Update(task); err != nil { | |||
| return err | |||
| } | |||
| task.Repo.Status = RepositoryReady | |||
| if _, err := sess.ID(task.RepoID).Cols("status").Update(task.Repo); err != nil { | |||
| return err | |||
| } | |||
| return sess.Commit() | |||
| } | |||
+ 20
- 15
modules/context/repo.go
View File
+ 3
- 18
modules/migrations/base/options.go
View File
+ 27
- 9
modules/migrations/gitea.go
View File
+ 4
- 3
modules/migrations/gitea_test.go
View File
+ 2
- 2
modules/migrations/github.go
View File
+ 9
- 3
modules/migrations/migrate.go
View File
+ 1
- 0
modules/setting/setting.go
View File
+ 25
- 0
modules/setting/task.go
View File
| @ -0,0 +1,25 @@ | |||
| // Copyright 2019 The Gitea Authors. All rights reserved. | |||
| // Use of this source code is governed by a MIT-style | |||
| // license that can be found in the LICENSE file. | |||
| package setting | |||
| var ( | |||
| // Task settings | |||
| Task = struct { | |||
| QueueType string | |||
| QueueLength int | |||
| QueueConnStr string | |||
| }{ | |||
| QueueType: ChannelQueueType, | |||
| QueueLength: 1000, | |||
| QueueConnStr: "addrs=127.0.0.1:6379 db=0", | |||
| } | |||
| ) | |||
| func newTaskService() { | |||
| sec := Cfg.Section("task") | |||
| Task.QueueType = sec.Key("QUEUE_TYPE").MustString(ChannelQueueType) | |||
| Task.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000) | |||
| Task.QueueConnStr = sec.Key("QUEUE_CONN_STR").MustString("addrs=127.0.0.1:6379 db=0") | |||
| } | |||
+ 12
- 4
modules/structs/repo.go
View File
+ 34
- 0
modules/structs/task.go
View File
| @ -0,0 +1,34 @@ | |||
| // Copyright 2019 Gitea. All rights reserved. | |||
| // Use of this source code is governed by a MIT-style | |||
| // license that can be found in the LICENSE file. | |||
| package structs | |||
| // TaskType defines task type | |||
| type TaskType int | |||
| // all kinds of task types | |||
| const ( | |||
| TaskTypeMigrateRepo TaskType = iota // migrate repository from external or local disk | |||
| ) | |||
| // Name returns the task type name | |||
| func (taskType TaskType) Name() string { | |||
| switch taskType { | |||
| case TaskTypeMigrateRepo: | |||
| return "Migrate Repository" | |||
| } | |||
| return "" | |||
| } | |||
| // TaskStatus defines task status | |||
| type TaskStatus int | |||
| // enumerate all the kinds of task status | |||
| const ( | |||
| TaskStatusQueue TaskStatus = iota // 0 task is queue | |||
| TaskStatusRunning // 1 task is running | |||
| TaskStatusStopped // 2 task is stopped | |||
| TaskStatusFailed // 3 task is failed | |||
| TaskStatusFinished // 4 task is finished | |||
| ) | |||
+ 120
- 0
modules/task/migrate.go
View File
| @ -0,0 +1,120 @@ | |||
| // Copyright 2019 Gitea. All rights reserved. | |||
| // Use of this source code is governed by a MIT-style | |||
| // license that can be found in the LICENSE file. | |||
| package task | |||
| import ( | |||
| "bytes" | |||
| "errors" | |||
| "fmt" | |||
| "strings" | |||
| "code.gitea.io/gitea/models" | |||
| "code.gitea.io/gitea/modules/log" | |||
| "code.gitea.io/gitea/modules/migrations" | |||
| "code.gitea.io/gitea/modules/notification" | |||
| "code.gitea.io/gitea/modules/structs" | |||
| "code.gitea.io/gitea/modules/timeutil" | |||
| "code.gitea.io/gitea/modules/util" | |||
| ) | |||
| func handleCreateError(owner *models.User, err error, name string) error { | |||
| switch { | |||
| case models.IsErrReachLimitOfRepo(err): | |||
| return fmt.Errorf("You have already reached your limit of %d repositories", owner.MaxCreationLimit()) | |||
| case models.IsErrRepoAlreadyExist(err): | |||
| return errors.New("The repository name is already used") | |||
| case models.IsErrNameReserved(err): | |||
| return fmt.Errorf("The repository name '%s' is reserved", err.(models.ErrNameReserved).Name) | |||
| case models.IsErrNamePatternNotAllowed(err): | |||
| return fmt.Errorf("The pattern '%s' is not allowed in a repository name", err.(models.ErrNamePatternNotAllowed).Pattern) | |||
| default: | |||
| return err | |||
| } | |||
| } | |||
| func runMigrateTask(t *models.Task) (err error) { | |||
| defer func() { | |||
| if e := recover(); e != nil { | |||
| var buf bytes.Buffer | |||
| fmt.Fprintf(&buf, "Handler crashed with error: %v", log.Stack(2)) | |||
| err = errors.New(buf.String()) | |||
| } | |||
| if err == nil { | |||
| err = models.FinishMigrateTask(t) | |||
| if err == nil { | |||
| notification.NotifyMigrateRepository(t.Doer, t.Owner, t.Repo) | |||
| return | |||
| } | |||
| log.Error("FinishMigrateTask failed: %s", err.Error()) | |||
| } | |||
| t.EndTime = timeutil.TimeStampNow() | |||
| t.Status = structs.TaskStatusFailed | |||
| t.Errors = err.Error() | |||
| if err := t.UpdateCols("status", "errors", "end_time"); err != nil { | |||
| log.Error("Task UpdateCols failed: %s", err.Error()) | |||
| } | |||
| if t.Repo != nil { | |||
| if errDelete := models.DeleteRepository(t.Doer, t.OwnerID, t.Repo.ID); errDelete != nil { | |||
| log.Error("DeleteRepository: %v", errDelete) | |||
| } | |||
| } | |||
| }() | |||
| if err := t.LoadRepo(); err != nil { | |||
| return err | |||
| } | |||
| // if repository is ready, then just finsih the task | |||
| if t.Repo.Status == models.RepositoryReady { | |||
| return nil | |||
| } | |||
| if err := t.LoadDoer(); err != nil { | |||
| return err | |||
| } | |||
| if err := t.LoadOwner(); err != nil { | |||
| return err | |||
| } | |||
| t.StartTime = timeutil.TimeStampNow() | |||
| t.Status = structs.TaskStatusRunning | |||
| if err := t.UpdateCols("start_time", "status"); err != nil { | |||
| return err | |||
| } | |||
| var opts *structs.MigrateRepoOption | |||
| opts, err = t.MigrateConfig() | |||
| if err != nil { | |||
| return err | |||
| } | |||
| opts.MigrateToRepoID = t.RepoID | |||
| repo, err := migrations.MigrateRepository(t.Doer, t.Owner.Name, *opts) | |||
| if err == nil { | |||
| notification.NotifyMigrateRepository(t.Doer, t.Owner, repo) | |||
| log.Trace("Repository migrated [%d]: %s/%s", repo.ID, t.Owner.Name, repo.Name) | |||
| return nil | |||
| } | |||
| if models.IsErrRepoAlreadyExist(err) { | |||
| return errors.New("The repository name is already used") | |||
| } | |||
| // remoteAddr may contain credentials, so we sanitize it | |||
| err = util.URLSanitizedError(err, opts.CloneAddr) | |||
| if strings.Contains(err.Error(), "Authentication failed") || | |||
| strings.Contains(err.Error(), "could not read Username") { | |||
| return fmt.Errorf("Authentication failed: %v", err.Error()) | |||
| } else if strings.Contains(err.Error(), "fatal:") { | |||
| return fmt.Errorf("Migration failed: %v", err.Error()) | |||
| } | |||
| return handleCreateError(t.Owner, err, "MigratePost") | |||
| } | |||
+ 14
- 0
modules/task/queue.go
View File
| @ -0,0 +1,14 @@ | |||
| // Copyright 2019 Gitea. All rights reserved. | |||
| // Use of this source code is governed by a MIT-style | |||
| // license that can be found in the LICENSE file. | |||
| package task | |||
| import "code.gitea.io/gitea/models" | |||
| // Queue defines an interface to run task queue | |||
| type Queue interface { | |||
| Run() error | |||
| Push(*models.Task) error | |||
| Stop() | |||
| } | |||
+ 48
- 0
modules/task/queue_channel.go
View File
| @ -0,0 +1,48 @@ | |||
| // Copyright 2019 The Gitea Authors. All rights reserved. | |||
| // Use of this source code is governed by a MIT-style | |||
| // license that can be found in the LICENSE file. | |||
| package task | |||
| import ( | |||
| "code.gitea.io/gitea/models" | |||
| "code.gitea.io/gitea/modules/log" | |||
| ) | |||
| var ( | |||
| _ Queue = &ChannelQueue{} | |||
| ) | |||
| // ChannelQueue implements | |||
| type ChannelQueue struct { | |||
| queue chan *models.Task | |||
| } | |||
| // NewChannelQueue create a memory channel queue | |||
| func NewChannelQueue(queueLen int) *ChannelQueue { | |||
| return &ChannelQueue{ | |||
| queue: make(chan *models.Task, queueLen), | |||
| } | |||
| } | |||
| // Run starts to run the queue | |||
| func (c *ChannelQueue) Run() error { | |||
| for task := range c.queue { | |||
| err := Run(task) | |||
| if err != nil { | |||
| log.Error("Run task failed: %s", err.Error()) | |||
| } | |||
| } | |||
| return nil | |||
| } | |||
| // Push will push the task ID to queue | |||
| func (c *ChannelQueue) Push(task *models.Task) error { | |||
| c.queue <- task | |||
| return nil | |||
| } | |||
| // Stop stop the queue | |||
| func (c *ChannelQueue) Stop() { | |||
| close(c.queue) | |||
| } | |||
+ 130
- 0
modules/task/queue_redis.go
View File
| @ -0,0 +1,130 @@ | |||
| // Copyright 2019 The Gitea Authors. All rights reserved. | |||
| // Use of this source code is governed by a MIT-style | |||
| // license that can be found in the LICENSE file. | |||
| package task | |||
| import ( | |||
| "encoding/json" | |||
| "errors" | |||
| "strconv" | |||
| "strings" | |||
| "time" | |||
| "code.gitea.io/gitea/models" | |||
| "code.gitea.io/gitea/modules/log" | |||
| "github.com/go-redis/redis" | |||
| ) | |||
| var ( | |||
| _ Queue = &RedisQueue{} | |||
| ) | |||
| type redisClient interface { | |||
| RPush(key string, args ...interface{}) *redis.IntCmd | |||
| LPop(key string) *redis.StringCmd | |||
| Ping() *redis.StatusCmd | |||
| } | |||
| // RedisQueue redis queue | |||
| type RedisQueue struct { | |||
| client redisClient | |||
| queueName string | |||
| closeChan chan bool | |||
| } | |||
| func parseConnStr(connStr string) (addrs, password string, dbIdx int, err error) { | |||
| fields := strings.Fields(connStr) | |||
| for _, f := range fields { | |||
| items := strings.SplitN(f, "=", 2) | |||
| if len(items) < 2 { | |||
| continue | |||
| } | |||
| switch strings.ToLower(items[0]) { | |||
| case "addrs": | |||
| addrs = items[1] | |||
| case "password": | |||
| password = items[1] | |||
| case "db": | |||
| dbIdx, err = strconv.Atoi(items[1]) | |||
| if err != nil { | |||
| return | |||
| } | |||
| } | |||
| } | |||
| return | |||
| } | |||
| // NewRedisQueue creates single redis or cluster redis queue | |||
| func NewRedisQueue(addrs string, password string, dbIdx int) (*RedisQueue, error) { | |||
| dbs := strings.Split(addrs, ",") | |||
| var queue = RedisQueue{ | |||
| queueName: "task_queue", | |||
| closeChan: make(chan bool), | |||
| } | |||
| if len(dbs) == 0 { | |||
| return nil, errors.New("no redis host found") | |||
| } else if len(dbs) == 1 { | |||
| queue.client = redis.NewClient(&redis.Options{ | |||
| Addr: strings.TrimSpace(dbs[0]), // use default Addr | |||
| Password: password, // no password set | |||
| DB: dbIdx, // use default DB | |||
| }) | |||
| } else { | |||
| // cluster will ignore db | |||
| queue.client = redis.NewClusterClient(&redis.ClusterOptions{ | |||
| Addrs: dbs, | |||
| Password: password, | |||
| }) | |||
| } | |||
| if err := queue.client.Ping().Err(); err != nil { | |||
| return nil, err | |||
| } | |||
| return &queue, nil | |||
| } | |||
| // Run starts to run the queue | |||
| func (r *RedisQueue) Run() error { | |||
| for { | |||
| select { | |||
| case <-r.closeChan: | |||
| return nil | |||
| case <-time.After(time.Millisecond * 100): | |||
| } | |||
| bs, err := r.client.LPop(r.queueName).Bytes() | |||
| if err != nil { | |||
| if err != redis.Nil { | |||
| log.Error("LPop failed: %v", err) | |||
| } | |||
| time.Sleep(time.Millisecond * 100) | |||
| continue | |||
| } | |||
| var task models.Task | |||
| err = json.Unmarshal(bs, &task) | |||
| if err != nil { | |||
| log.Error("Unmarshal task failed: %s", err.Error()) | |||
| } else { | |||
| err = Run(&task) | |||
| if err != nil { | |||
| log.Error("Run task failed: %s", err.Error()) | |||
| } | |||
| } | |||
| } | |||
| } | |||
| // Push implements Queue | |||
| func (r *RedisQueue) Push(task *models.Task) error { | |||
| bs, err := json.Marshal(task) | |||
| if err != nil { | |||
| return err | |||
| } | |||
| return r.client.RPush(r.queueName, bs).Err() | |||
| } | |||
| // Stop stop the queue | |||
| func (r *RedisQueue) Stop() { | |||
| r.closeChan <- true | |||
| } | |||
+ 66
- 0
modules/task/task.go
View File
| @ -0,0 +1,66 @@ | |||
| // Copyright 2019 Gitea. All rights reserved. | |||
| // Use of this source code is governed by a MIT-style | |||
| // license that can be found in the LICENSE file. | |||
| package task | |||
| import ( | |||
| "fmt" | |||
| "code.gitea.io/gitea/models" | |||
| "code.gitea.io/gitea/modules/log" | |||
| "code.gitea.io/gitea/modules/migrations/base" | |||
| "code.gitea.io/gitea/modules/setting" | |||
| "code.gitea.io/gitea/modules/structs" | |||
| ) | |||
| // taskQueue is a global queue of tasks | |||
| var taskQueue Queue | |||
| // Run a task | |||
| func Run(t *models.Task) error { | |||
| switch t.Type { | |||
| case structs.TaskTypeMigrateRepo: | |||
| return runMigrateTask(t) | |||
| default: | |||
| return fmt.Errorf("Unknow task type: %d", t.Type) | |||
| } | |||
| } | |||
| // Init will start the service to get all unfinished tasks and run them | |||
| func Init() error { | |||
| switch setting.Task.QueueType { | |||
| case setting.ChannelQueueType: | |||
| taskQueue = NewChannelQueue(setting.Task.QueueLength) | |||
| case setting.RedisQueueType: | |||
| var err error | |||
| addrs, pass, idx, err := parseConnStr(setting.Task.QueueConnStr) | |||
| if err != nil { | |||
| return err | |||
| } | |||
| taskQueue, err = NewRedisQueue(addrs, pass, idx) | |||
| if err != nil { | |||
| return err | |||
| } | |||
| default: | |||
| return fmt.Errorf("Unsupported task queue type: %v", setting.Task.QueueType) | |||
| } | |||
| go func() { | |||
| if err := taskQueue.Run(); err != nil { | |||
| log.Error("taskQueue.Run end failed: %v", err) | |||
| } | |||
| }() | |||
| return nil | |||
| } | |||
| // MigrateRepository add migration repository to task | |||
| func MigrateRepository(doer, u *models.User, opts base.MigrateOptions) error { | |||
| task, err := models.CreateMigrateTask(doer, u, opts) | |||
| if err != nil { | |||
| return err | |||
| } | |||
| return taskQueue.Push(task) | |||
| } | |||
+ 2
- 0
options/locale/locale_en-US.ini
View File
BIN
View File
+ 36
- 0
public/js/index.js
View File
+ 2
- 2
routers/api/v1/repo/repo.go
View File
+ 4
- 0
routers/init.go
View File
+ 9
- 0
routers/private/serv.go
View File
+ 62
- 41
routers/repo/repo.go
View File
+ 30
- 0
routers/repo/view.go
View File
+ 2
- 0
routers/routes/routes.go
View File
+ 20
- 9
services/mirror/mirror_test.go
View File
+ 73
- 71
templates/repo/header.tmpl
View File
+ 31
- 0
templates/repo/migrating.tmpl
View File
| @ -0,0 +1,31 @@ | |||
| {{template "base/head" .}} | |||
| <div class="repository quickstart"> | |||
| {{template "repo/header" .}} | |||
| <div class="ui container"> | |||
| <div class="ui grid"> | |||
| <div class="sixteen wide column content"> | |||
| {{template "base/alert" .}} | |||
| <div class="home"> | |||
| <div class="ui stackable middle very relaxed page grid"> | |||
| <div id="repo_migrating" class="sixteen wide center aligned centered column" repo="{{.Repo.Repository.FullName}}"> | |||
| <div> | |||
| <img src="{{AppSubUrl}}/img/loading.png"/> | |||
| </div> | |||
| </div> | |||
| </div> | |||
| <div class="ui stackable middle very relaxed page grid"> | |||
| <div class="sixteen wide center aligned centered column"> | |||
| <div id="repo_migrating_progress"> | |||
| <p>{{.i18n.Tr "repo.migrate.migrating" .CloneAddr | Safe}}</p> | |||
| </div> | |||
| <div id="repo_migrating_failed"> | |||
| <p>{{.i18n.Tr "repo.migrate.migrating_failed" .CloneAddr | Safe}}</p> | |||
| </div> | |||
| </div> | |||
| </div> | |||
| </div> | |||
| </div> | |||
| </div> | |||
| </div> | |||
| </div> | |||
| {{template "base/footer" .}} | |||