You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

835 lines
22 KiB

Add a storage layer for attachments (#11387) * Add a storage layer for attachments * Fix some bug * fix test * Fix copyright head and lint * Fix bug * Add setting for minio and flags for migrate-storage * Add documents * fix lint * Add test for minio store type on attachments * fix test * fix test * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Add warning when storage migrated successfully * Fix drone * fix test * rebase * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * remove log on xorm * Fi download bug * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * Add URL function to serve attachments directly from S3/Minio * Add ability to enable/disable redirection in attachment configuration * Fix typo * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * don't change unrelated files * Fix lint * Fix build * update go.mod and go.sum * Use github.com/minio/minio-go/v6 * Remove unused function * Upgrade minio to v7 and some other improvements * fix lint * Fix go mod Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Tyler <tystuyfzand@gmail.com>
3 years ago
  1. // Copyright 2019 The Gitea Authors. All rights reserved.
  2. // Copyright 2018 Jonas Franz. All rights reserved.
  3. // Use of this source code is governed by a MIT-style
  4. // license that can be found in the LICENSE file.
  5. package migrations
  6. import (
  7. "bytes"
  8. "context"
  9. "fmt"
  10. "io"
  11. "net/http"
  12. "net/url"
  13. "os"
  14. "path/filepath"
  15. "strings"
  16. "sync"
  17. "time"
  18. "code.gitea.io/gitea/models"
  19. "code.gitea.io/gitea/modules/git"
  20. "code.gitea.io/gitea/modules/log"
  21. "code.gitea.io/gitea/modules/migrations/base"
  22. "code.gitea.io/gitea/modules/repository"
  23. repo_module "code.gitea.io/gitea/modules/repository"
  24. "code.gitea.io/gitea/modules/setting"
  25. "code.gitea.io/gitea/modules/storage"
  26. "code.gitea.io/gitea/modules/structs"
  27. "code.gitea.io/gitea/modules/timeutil"
  28. gouuid "github.com/google/uuid"
  29. )
  30. var (
  31. _ base.Uploader = &GiteaLocalUploader{}
  32. )
  33. // GiteaLocalUploader implements an Uploader to gitea sites
  34. type GiteaLocalUploader struct {
  35. ctx context.Context
  36. doer *models.User
  37. repoOwner string
  38. repoName string
  39. repo *models.Repository
  40. labels sync.Map
  41. milestones sync.Map
  42. issues sync.Map
  43. gitRepo *git.Repository
  44. prHeadCache map[string]struct{}
  45. userMap map[int64]int64 // external user id mapping to user id
  46. prCache map[int64]*models.PullRequest
  47. gitServiceType structs.GitServiceType
  48. }
  49. // NewGiteaLocalUploader creates an gitea Uploader via gitea API v1
  50. func NewGiteaLocalUploader(ctx context.Context, doer *models.User, repoOwner, repoName string) *GiteaLocalUploader {
  51. return &GiteaLocalUploader{
  52. ctx: ctx,
  53. doer: doer,
  54. repoOwner: repoOwner,
  55. repoName: repoName,
  56. prHeadCache: make(map[string]struct{}),
  57. userMap: make(map[int64]int64),
  58. prCache: make(map[int64]*models.PullRequest),
  59. }
  60. }
  61. // MaxBatchInsertSize returns the table's max batch insert size
  62. func (g *GiteaLocalUploader) MaxBatchInsertSize(tp string) int {
  63. switch tp {
  64. case "issue":
  65. return models.MaxBatchInsertSize(new(models.Issue))
  66. case "comment":
  67. return models.MaxBatchInsertSize(new(models.Comment))
  68. case "milestone":
  69. return models.MaxBatchInsertSize(new(models.Milestone))
  70. case "label":
  71. return models.MaxBatchInsertSize(new(models.Label))
  72. case "release":
  73. return models.MaxBatchInsertSize(new(models.Release))
  74. case "pullrequest":
  75. return models.MaxBatchInsertSize(new(models.PullRequest))
  76. }
  77. return 10
  78. }
  79. // CreateRepo creates a repository
  80. func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error {
  81. owner, err := models.GetUserByName(g.repoOwner)
  82. if err != nil {
  83. return err
  84. }
  85. var remoteAddr = repo.CloneURL
  86. if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
  87. u, err := url.Parse(repo.CloneURL)
  88. if err != nil {
  89. return err
  90. }
  91. u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
  92. if len(opts.AuthToken) > 0 {
  93. u.User = url.UserPassword("oauth2", opts.AuthToken)
  94. }
  95. remoteAddr = u.String()
  96. }
  97. var r *models.Repository
  98. if opts.MigrateToRepoID <= 0 {
  99. r, err = repo_module.CreateRepository(g.doer, owner, models.CreateRepoOptions{
  100. Name: g.repoName,
  101. Description: repo.Description,
  102. OriginalURL: repo.OriginalURL,
  103. GitServiceType: opts.GitServiceType,
  104. IsPrivate: opts.Private,
  105. IsMirror: opts.Mirror,
  106. Status: models.RepositoryBeingMigrated,
  107. })
  108. } else {
  109. r, err = models.GetRepositoryByID(opts.MigrateToRepoID)
  110. }
  111. if err != nil {
  112. return err
  113. }
  114. r, err = repository.MigrateRepositoryGitData(g.doer, owner, r, base.MigrateOptions{
  115. RepoName: g.repoName,
  116. Description: repo.Description,
  117. OriginalURL: repo.OriginalURL,
  118. GitServiceType: opts.GitServiceType,
  119. Mirror: repo.IsMirror,
  120. CloneAddr: remoteAddr,
  121. Private: repo.IsPrivate,
  122. Wiki: opts.Wiki,
  123. Releases: opts.Releases, // if didn't get releases, then sync them from tags
  124. })
  125. g.repo = r
  126. if err != nil {
  127. return err
  128. }
  129. g.gitRepo, err = git.OpenRepository(r.RepoPath())
  130. return err
  131. }
  132. // Close closes this uploader
  133. func (g *GiteaLocalUploader) Close() {
  134. if g.gitRepo != nil {
  135. g.gitRepo.Close()
  136. }
  137. }
  138. // CreateTopics creates topics
  139. func (g *GiteaLocalUploader) CreateTopics(topics ...string) error {
  140. return models.SaveTopics(g.repo.ID, topics...)
  141. }
  142. // CreateMilestones creates milestones
  143. func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) error {
  144. var mss = make([]*models.Milestone, 0, len(milestones))
  145. for _, milestone := range milestones {
  146. var deadline timeutil.TimeStamp
  147. if milestone.Deadline != nil {
  148. deadline = timeutil.TimeStamp(milestone.Deadline.Unix())
  149. }
  150. if deadline == 0 {
  151. deadline = timeutil.TimeStamp(time.Date(9999, 1, 1, 0, 0, 0, 0, setting.DefaultUILocation).Unix())
  152. }
  153. var ms = models.Milestone{
  154. RepoID: g.repo.ID,
  155. Name: milestone.Title,
  156. Content: milestone.Description,
  157. IsClosed: milestone.State == "closed",
  158. DeadlineUnix: deadline,
  159. }
  160. if ms.IsClosed && milestone.Closed != nil {
  161. ms.ClosedDateUnix = timeutil.TimeStamp(milestone.Closed.Unix())
  162. }
  163. mss = append(mss, &ms)
  164. }
  165. err := models.InsertMilestones(mss...)
  166. if err != nil {
  167. return err
  168. }
  169. for _, ms := range mss {
  170. g.milestones.Store(ms.Name, ms.ID)
  171. }
  172. return nil
  173. }
  174. // CreateLabels creates labels
  175. func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error {
  176. var lbs = make([]*models.Label, 0, len(labels))
  177. for _, label := range labels {
  178. lbs = append(lbs, &models.Label{
  179. RepoID: g.repo.ID,
  180. Name: label.Name,
  181. Description: label.Description,
  182. Color: fmt.Sprintf("#%s", label.Color),
  183. })
  184. }
  185. err := models.NewLabels(lbs...)
  186. if err != nil {
  187. return err
  188. }
  189. for _, lb := range lbs {
  190. g.labels.Store(lb.Name, lb)
  191. }
  192. return nil
  193. }
  194. // CreateReleases creates releases
  195. func (g *GiteaLocalUploader) CreateReleases(downloader base.Downloader, releases ...*base.Release) error {
  196. var rels = make([]*models.Release, 0, len(releases))
  197. for _, release := range releases {
  198. var rel = models.Release{
  199. RepoID: g.repo.ID,
  200. TagName: release.TagName,
  201. LowerTagName: strings.ToLower(release.TagName),
  202. Target: release.TargetCommitish,
  203. Title: release.Name,
  204. Sha1: release.TargetCommitish,
  205. Note: release.Body,
  206. IsDraft: release.Draft,
  207. IsPrerelease: release.Prerelease,
  208. IsTag: false,
  209. CreatedUnix: timeutil.TimeStamp(release.Created.Unix()),
  210. }
  211. userid, ok := g.userMap[release.PublisherID]
  212. tp := g.gitServiceType.Name()
  213. if !ok && tp != "" {
  214. var err error
  215. userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", release.PublisherID))
  216. if err != nil {
  217. log.Error("GetUserIDByExternalUserID: %v", err)
  218. }
  219. if userid > 0 {
  220. g.userMap[release.PublisherID] = userid
  221. }
  222. }
  223. if userid > 0 {
  224. rel.PublisherID = userid
  225. } else {
  226. rel.PublisherID = g.doer.ID
  227. rel.OriginalAuthor = release.PublisherName
  228. rel.OriginalAuthorID = release.PublisherID
  229. }
  230. // calc NumCommits
  231. commit, err := g.gitRepo.GetCommit(rel.TagName)
  232. if err != nil {
  233. return fmt.Errorf("GetCommit: %v", err)
  234. }
  235. rel.NumCommits, err = commit.CommitsCount()
  236. if err != nil {
  237. return fmt.Errorf("CommitsCount: %v", err)
  238. }
  239. for _, asset := range release.Assets {
  240. var attach = models.Attachment{
  241. UUID: gouuid.New().String(),
  242. Name: asset.Name,
  243. DownloadCount: int64(*asset.DownloadCount),
  244. Size: int64(*asset.Size),
  245. CreatedUnix: timeutil.TimeStamp(asset.Created.Unix()),
  246. }
  247. // download attachment
  248. err = func() error {
  249. rc, err := downloader.GetAsset(rel.TagName, asset.ID)
  250. if err != nil {
  251. return err
  252. }
  253. _, err = storage.Attachments.Save(attach.RelativePath(), rc)
  254. return err
  255. }()
  256. if err != nil {
  257. return err
  258. }
  259. rel.Attachments = append(rel.Attachments, &attach)
  260. }
  261. rels = append(rels, &rel)
  262. }
  263. return models.InsertReleases(rels...)
  264. }
  265. // SyncTags syncs releases with tags in the database
  266. func (g *GiteaLocalUploader) SyncTags() error {
  267. return repository.SyncReleasesWithTags(g.repo, g.gitRepo)
  268. }
  269. // CreateIssues creates issues
  270. func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error {
  271. var iss = make([]*models.Issue, 0, len(issues))
  272. for _, issue := range issues {
  273. var labels []*models.Label
  274. for _, label := range issue.Labels {
  275. lb, ok := g.labels.Load(label.Name)
  276. if ok {
  277. labels = append(labels, lb.(*models.Label))
  278. }
  279. }
  280. var milestoneID int64
  281. if issue.Milestone != "" {
  282. milestone, ok := g.milestones.Load(issue.Milestone)
  283. if ok {
  284. milestoneID = milestone.(int64)
  285. }
  286. }
  287. var is = models.Issue{
  288. RepoID: g.repo.ID,
  289. Repo: g.repo,
  290. Index: issue.Number,
  291. Title: issue.Title,
  292. Content: issue.Content,
  293. IsClosed: issue.State == "closed",
  294. IsLocked: issue.IsLocked,
  295. MilestoneID: milestoneID,
  296. Labels: labels,
  297. CreatedUnix: timeutil.TimeStamp(issue.Created.Unix()),
  298. UpdatedUnix: timeutil.TimeStamp(issue.Updated.Unix()),
  299. }
  300. userid, ok := g.userMap[issue.PosterID]
  301. tp := g.gitServiceType.Name()
  302. if !ok && tp != "" {
  303. var err error
  304. userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", issue.PosterID))
  305. if err != nil {
  306. log.Error("GetUserIDByExternalUserID: %v", err)
  307. }
  308. if userid > 0 {
  309. g.userMap[issue.PosterID] = userid
  310. }
  311. }
  312. if userid > 0 {
  313. is.PosterID = userid
  314. } else {
  315. is.PosterID = g.doer.ID
  316. is.OriginalAuthor = issue.PosterName
  317. is.OriginalAuthorID = issue.PosterID
  318. }
  319. if issue.Closed != nil {
  320. is.ClosedUnix = timeutil.TimeStamp(issue.Closed.Unix())
  321. }
  322. // add reactions
  323. for _, reaction := range issue.Reactions {
  324. userid, ok := g.userMap[reaction.UserID]
  325. if !ok && tp != "" {
  326. var err error
  327. userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", reaction.UserID))
  328. if err != nil {
  329. log.Error("GetUserIDByExternalUserID: %v", err)
  330. }
  331. if userid > 0 {
  332. g.userMap[reaction.UserID] = userid
  333. }
  334. }
  335. var res = models.Reaction{
  336. Type: reaction.Content,
  337. CreatedUnix: timeutil.TimeStampNow(),
  338. }
  339. if userid > 0 {
  340. res.UserID = userid
  341. } else {
  342. res.UserID = g.doer.ID
  343. res.OriginalAuthorID = reaction.UserID
  344. res.OriginalAuthor = reaction.UserName
  345. }
  346. is.Reactions = append(is.Reactions, &res)
  347. }
  348. iss = append(iss, &is)
  349. }
  350. if len(iss) > 0 {
  351. if err := models.InsertIssues(iss...); err != nil {
  352. return err
  353. }
  354. for _, is := range iss {
  355. g.issues.Store(is.Index, is.ID)
  356. }
  357. }
  358. return nil
  359. }
  360. // CreateComments creates comments of issues
  361. func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error {
  362. var cms = make([]*models.Comment, 0, len(comments))
  363. for _, comment := range comments {
  364. var issueID int64
  365. if issueIDStr, ok := g.issues.Load(comment.IssueIndex); !ok {
  366. issue, err := models.GetIssueByIndex(g.repo.ID, comment.IssueIndex)
  367. if err != nil {
  368. return err
  369. }
  370. issueID = issue.ID
  371. g.issues.Store(comment.IssueIndex, issueID)
  372. } else {
  373. issueID = issueIDStr.(int64)
  374. }
  375. userid, ok := g.userMap[comment.PosterID]
  376. tp := g.gitServiceType.Name()
  377. if !ok && tp != "" {
  378. var err error
  379. userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", comment.PosterID))
  380. if err != nil {
  381. log.Error("GetUserIDByExternalUserID: %v", err)
  382. }
  383. if userid > 0 {
  384. g.userMap[comment.PosterID] = userid
  385. }
  386. }
  387. cm := models.Comment{
  388. IssueID: issueID,
  389. Type: models.CommentTypeComment,
  390. Content: comment.Content,
  391. CreatedUnix: timeutil.TimeStamp(comment.Created.Unix()),
  392. UpdatedUnix: timeutil.TimeStamp(comment.Updated.Unix()),
  393. }
  394. if userid > 0 {
  395. cm.PosterID = userid
  396. } else {
  397. cm.PosterID = g.doer.ID
  398. cm.OriginalAuthor = comment.PosterName
  399. cm.OriginalAuthorID = comment.PosterID
  400. }
  401. // add reactions
  402. for _, reaction := range comment.Reactions {
  403. userid, ok := g.userMap[reaction.UserID]
  404. if !ok && tp != "" {
  405. var err error
  406. userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", reaction.UserID))
  407. if err != nil {
  408. log.Error("GetUserIDByExternalUserID: %v", err)
  409. }
  410. if userid > 0 {
  411. g.userMap[reaction.UserID] = userid
  412. }
  413. }
  414. var res = models.Reaction{
  415. Type: reaction.Content,
  416. CreatedUnix: timeutil.TimeStampNow(),
  417. }
  418. if userid > 0 {
  419. res.UserID = userid
  420. } else {
  421. res.UserID = g.doer.ID
  422. res.OriginalAuthorID = reaction.UserID
  423. res.OriginalAuthor = reaction.UserName
  424. }
  425. cm.Reactions = append(cm.Reactions, &res)
  426. }
  427. cms = append(cms, &cm)
  428. }
  429. if len(cms) == 0 {
  430. return nil
  431. }
  432. return models.InsertIssueComments(cms)
  433. }
  434. // CreatePullRequests creates pull requests
  435. func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error {
  436. var gprs = make([]*models.PullRequest, 0, len(prs))
  437. for _, pr := range prs {
  438. gpr, err := g.newPullRequest(pr)
  439. if err != nil {
  440. return err
  441. }
  442. userid, ok := g.userMap[pr.PosterID]
  443. tp := g.gitServiceType.Name()
  444. if !ok && tp != "" {
  445. var err error
  446. userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", pr.PosterID))
  447. if err != nil {
  448. log.Error("GetUserIDByExternalUserID: %v", err)
  449. }
  450. if userid > 0 {
  451. g.userMap[pr.PosterID] = userid
  452. }
  453. }
  454. if userid > 0 {
  455. gpr.Issue.PosterID = userid
  456. } else {
  457. gpr.Issue.PosterID = g.doer.ID
  458. gpr.Issue.OriginalAuthor = pr.PosterName
  459. gpr.Issue.OriginalAuthorID = pr.PosterID
  460. }
  461. gprs = append(gprs, gpr)
  462. }
  463. if err := models.InsertPullRequests(gprs...); err != nil {
  464. return err
  465. }
  466. for _, pr := range gprs {
  467. g.issues.Store(pr.Issue.Index, pr.Issue.ID)
  468. }
  469. return nil
  470. }
  471. func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullRequest, error) {
  472. var labels []*models.Label
  473. for _, label := range pr.Labels {
  474. lb, ok := g.labels.Load(label.Name)
  475. if ok {
  476. labels = append(labels, lb.(*models.Label))
  477. }
  478. }
  479. var milestoneID int64
  480. if pr.Milestone != "" {
  481. milestone, ok := g.milestones.Load(pr.Milestone)
  482. if ok {
  483. milestoneID = milestone.(int64)
  484. }
  485. }
  486. // download patch file
  487. err := func() error {
  488. resp, err := http.Get(pr.PatchURL)
  489. if err != nil {
  490. return err
  491. }
  492. defer resp.Body.Close()
  493. pullDir := filepath.Join(g.repo.RepoPath(), "pulls")
  494. if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
  495. return err
  496. }
  497. f, err := os.Create(filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number)))
  498. if err != nil {
  499. return err
  500. }
  501. defer f.Close()
  502. _, err = io.Copy(f, resp.Body)
  503. return err
  504. }()
  505. if err != nil {
  506. return nil, err
  507. }
  508. // set head information
  509. pullHead := filepath.Join(g.repo.RepoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number))
  510. if err := os.MkdirAll(pullHead, os.ModePerm); err != nil {
  511. return nil, err
  512. }
  513. p, err := os.Create(filepath.Join(pullHead, "head"))
  514. if err != nil {
  515. return nil, err
  516. }
  517. _, err = p.WriteString(pr.Head.SHA)
  518. p.Close()
  519. if err != nil {
  520. return nil, err
  521. }
  522. var head = "unknown repository"
  523. if pr.IsForkPullRequest() && pr.State != "closed" {
  524. if pr.Head.OwnerName != "" {
  525. remote := pr.Head.OwnerName
  526. _, ok := g.prHeadCache[remote]
  527. if !ok {
  528. // git remote add
  529. err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true)
  530. if err != nil {
  531. log.Error("AddRemote failed: %s", err)
  532. } else {
  533. g.prHeadCache[remote] = struct{}{}
  534. ok = true
  535. }
  536. }
  537. if ok {
  538. _, err = git.NewCommand("fetch", remote, pr.Head.Ref).RunInDir(g.repo.RepoPath())
  539. if err != nil {
  540. log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
  541. } else {
  542. headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref)
  543. if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil {
  544. return nil, err
  545. }
  546. b, err := os.Create(headBranch)
  547. if err != nil {
  548. return nil, err
  549. }
  550. _, err = b.WriteString(pr.Head.SHA)
  551. b.Close()
  552. if err != nil {
  553. return nil, err
  554. }
  555. head = pr.Head.OwnerName + "/" + pr.Head.Ref
  556. }
  557. }
  558. }
  559. } else {
  560. head = pr.Head.Ref
  561. }
  562. var issue = models.Issue{
  563. RepoID: g.repo.ID,
  564. Repo: g.repo,
  565. Title: pr.Title,
  566. Index: pr.Number,
  567. Content: pr.Content,
  568. MilestoneID: milestoneID,
  569. IsPull: true,
  570. IsClosed: pr.State == "closed",
  571. IsLocked: pr.IsLocked,
  572. Labels: labels,
  573. CreatedUnix: timeutil.TimeStamp(pr.Created.Unix()),
  574. UpdatedUnix: timeutil.TimeStamp(pr.Updated.Unix()),
  575. }
  576. tp := g.gitServiceType.Name()
  577. userid, ok := g.userMap[pr.PosterID]
  578. if !ok && tp != "" {
  579. var err error
  580. userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", pr.PosterID))
  581. if err != nil {
  582. log.Error("GetUserIDByExternalUserID: %v", err)
  583. }
  584. if userid > 0 {
  585. g.userMap[pr.PosterID] = userid
  586. }
  587. }
  588. if userid > 0 {
  589. issue.PosterID = userid
  590. } else {
  591. issue.PosterID = g.doer.ID
  592. issue.OriginalAuthor = pr.PosterName
  593. issue.OriginalAuthorID = pr.PosterID
  594. }
  595. // add reactions
  596. for _, reaction := range pr.Reactions {
  597. userid, ok := g.userMap[reaction.UserID]
  598. if !ok && tp != "" {
  599. var err error
  600. userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", reaction.UserID))
  601. if err != nil {
  602. log.Error("GetUserIDByExternalUserID: %v", err)
  603. }
  604. if userid > 0 {
  605. g.userMap[reaction.UserID] = userid
  606. }
  607. }
  608. var res = models.Reaction{
  609. Type: reaction.Content,
  610. CreatedUnix: timeutil.TimeStampNow(),
  611. }
  612. if userid > 0 {
  613. res.UserID = userid
  614. } else {
  615. res.UserID = g.doer.ID
  616. res.OriginalAuthorID = reaction.UserID
  617. res.OriginalAuthor = reaction.UserName
  618. }
  619. issue.Reactions = append(issue.Reactions, &res)
  620. }
  621. var pullRequest = models.PullRequest{
  622. HeadRepoID: g.repo.ID,
  623. HeadBranch: head,
  624. BaseRepoID: g.repo.ID,
  625. BaseBranch: pr.Base.Ref,
  626. MergeBase: pr.Base.SHA,
  627. Index: pr.Number,
  628. HasMerged: pr.Merged,
  629. Issue: &issue,
  630. }
  631. if pullRequest.Issue.IsClosed && pr.Closed != nil {
  632. pullRequest.Issue.ClosedUnix = timeutil.TimeStamp(pr.Closed.Unix())
  633. }
  634. if pullRequest.HasMerged && pr.MergedTime != nil {
  635. pullRequest.MergedUnix = timeutil.TimeStamp(pr.MergedTime.Unix())
  636. pullRequest.MergedCommitID = pr.MergeCommitSHA
  637. pullRequest.MergerID = g.doer.ID
  638. }
  639. // TODO: assignees
  640. return &pullRequest, nil
  641. }
  642. func convertReviewState(state string) models.ReviewType {
  643. switch state {
  644. case base.ReviewStatePending:
  645. return models.ReviewTypePending
  646. case base.ReviewStateApproved:
  647. return models.ReviewTypeApprove
  648. case base.ReviewStateChangesRequested:
  649. return models.ReviewTypeReject
  650. case base.ReviewStateCommented:
  651. return models.ReviewTypeComment
  652. default:
  653. return models.ReviewTypePending
  654. }
  655. }
  656. // CreateReviews create pull request reviews
  657. func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error {
  658. var cms = make([]*models.Review, 0, len(reviews))
  659. for _, review := range reviews {
  660. var issueID int64
  661. if issueIDStr, ok := g.issues.Load(review.IssueIndex); !ok {
  662. issue, err := models.GetIssueByIndex(g.repo.ID, review.IssueIndex)
  663. if err != nil {
  664. return err
  665. }
  666. issueID = issue.ID
  667. g.issues.Store(review.IssueIndex, issueID)
  668. } else {
  669. issueID = issueIDStr.(int64)
  670. }
  671. userid, ok := g.userMap[review.ReviewerID]
  672. tp := g.gitServiceType.Name()
  673. if !ok && tp != "" {
  674. var err error
  675. userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", review.ReviewerID))
  676. if err != nil {
  677. log.Error("GetUserIDByExternalUserID: %v", err)
  678. }
  679. if userid > 0 {
  680. g.userMap[review.ReviewerID] = userid
  681. }
  682. }
  683. var cm = models.Review{
  684. Type: convertReviewState(review.State),
  685. IssueID: issueID,
  686. Content: review.Content,
  687. Official: review.Official,
  688. CreatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
  689. UpdatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
  690. }
  691. if userid > 0 {
  692. cm.ReviewerID = userid
  693. } else {
  694. cm.ReviewerID = g.doer.ID
  695. cm.OriginalAuthor = review.ReviewerName
  696. cm.OriginalAuthorID = review.ReviewerID
  697. }
  698. // get pr
  699. pr, ok := g.prCache[issueID]
  700. if !ok {
  701. var err error
  702. pr, err = models.GetPullRequestByIssueIDWithNoAttributes(issueID)
  703. if err != nil {
  704. return err
  705. }
  706. g.prCache[issueID] = pr
  707. }
  708. for _, comment := range review.Comments {
  709. _, _, line, _ := git.ParseDiffHunkString(comment.DiffHunk)
  710. headCommitID, err := g.gitRepo.GetRefCommitID(pr.GetGitRefName())
  711. if err != nil {
  712. return fmt.Errorf("GetRefCommitID[%s]: %v", pr.GetGitRefName(), err)
  713. }
  714. var patch string
  715. patchBuf := new(bytes.Buffer)
  716. if err := git.GetRepoRawDiffForFile(g.gitRepo, pr.MergeBase, headCommitID, git.RawDiffNormal, comment.TreePath, patchBuf); err != nil {
  717. // We should ignore the error since the commit maybe removed when force push to the pull request
  718. log.Warn("GetRepoRawDiffForFile failed when migrating [%s, %s, %s, %s]: %v", g.gitRepo.Path, pr.MergeBase, headCommitID, comment.TreePath, err)
  719. } else {
  720. patch = git.CutDiffAroundLine(patchBuf, int64((&models.Comment{Line: int64(line + comment.Position - 1)}).UnsignedLine()), line < 0, setting.UI.CodeCommentLines)
  721. }
  722. var c = models.Comment{
  723. Type: models.CommentTypeCode,
  724. PosterID: comment.PosterID,
  725. IssueID: issueID,
  726. Content: comment.Content,
  727. Line: int64(line + comment.Position - 1),
  728. TreePath: comment.TreePath,
  729. CommitSHA: comment.CommitID,
  730. Patch: patch,
  731. CreatedUnix: timeutil.TimeStamp(comment.CreatedAt.Unix()),
  732. UpdatedUnix: timeutil.TimeStamp(comment.UpdatedAt.Unix()),
  733. }
  734. if userid > 0 {
  735. c.PosterID = userid
  736. } else {
  737. c.PosterID = g.doer.ID
  738. c.OriginalAuthor = review.ReviewerName
  739. c.OriginalAuthorID = review.ReviewerID
  740. }
  741. cm.Comments = append(cm.Comments, &c)
  742. }
  743. cms = append(cms, &cm)
  744. }
  745. return models.InsertReviews(cms)
  746. }
  747. // Rollback when migrating failed, this will rollback all the changes.
  748. func (g *GiteaLocalUploader) Rollback() error {
  749. if g.repo != nil && g.repo.ID > 0 {
  750. if err := models.DeleteRepository(g.doer, g.repo.OwnerID, g.repo.ID); err != nil {
  751. return err
  752. }
  753. }
  754. return nil
  755. }