You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

171 lines
4.4 KiB

Add a storage layer for attachments (#11387) * Add a storage layer for attachments * Fix some bug * fix test * Fix copyright head and lint * Fix bug * Add setting for minio and flags for migrate-storage * Add documents * fix lint * Add test for minio store type on attachments * fix test * fix test * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Add warning when storage migrated successfully * Fix drone * fix test * rebase * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * remove log on xorm * Fi download bug * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * Add URL function to serve attachments directly from S3/Minio * Add ability to enable/disable redirection in attachment configuration * Fix typo * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * don't change unrelated files * Fix lint * Fix build * update go.mod and go.sum * Use github.com/minio/minio-go/v6 * Remove unused function * Upgrade minio to v7 and some other improvements * fix lint * Fix go mod Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Tyler <tystuyfzand@gmail.com>
3 years ago
Add a storage layer for attachments (#11387) * Add a storage layer for attachments * Fix some bug * fix test * Fix copyright head and lint * Fix bug * Add setting for minio and flags for migrate-storage * Add documents * fix lint * Add test for minio store type on attachments * fix test * fix test * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Add warning when storage migrated successfully * Fix drone * fix test * rebase * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * remove log on xorm * Fi download bug * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * Add URL function to serve attachments directly from S3/Minio * Add ability to enable/disable redirection in attachment configuration * Fix typo * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * don't change unrelated files * Fix lint * Fix build * update go.mod and go.sum * Use github.com/minio/minio-go/v6 * Remove unused function * Upgrade minio to v7 and some other improvements * fix lint * Fix go mod Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Tyler <tystuyfzand@gmail.com>
3 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
  1. // Copyright 2014 The Gogs Authors. All rights reserved.
  2. // Copyright 2018 The Gitea Authors. All rights reserved.
  3. // Use of this source code is governed by a MIT-style
  4. // license that can be found in the LICENSE file.
  5. package repo
  6. import (
  7. "fmt"
  8. "io"
  9. "path"
  10. "strings"
  11. "code.gitea.io/gitea/modules/base"
  12. "code.gitea.io/gitea/modules/charset"
  13. "code.gitea.io/gitea/modules/context"
  14. "code.gitea.io/gitea/modules/git"
  15. "code.gitea.io/gitea/modules/lfs"
  16. "code.gitea.io/gitea/modules/log"
  17. )
  18. // ServeData download file from io.Reader
  19. func ServeData(ctx *context.Context, name string, reader io.Reader) error {
  20. buf := make([]byte, 1024)
  21. n, err := reader.Read(buf)
  22. if err != nil && err != io.EOF {
  23. return err
  24. }
  25. if n >= 0 {
  26. buf = buf[:n]
  27. }
  28. ctx.Resp.Header().Set("Cache-Control", "public,max-age=86400")
  29. name = path.Base(name)
  30. // Google Chrome dislike commas in filenames, so let's change it to a space
  31. name = strings.Replace(name, ",", " ", -1)
  32. if base.IsTextFile(buf) || ctx.QueryBool("render") {
  33. cs, err := charset.DetectEncoding(buf)
  34. if err != nil {
  35. log.Error("Detect raw file %s charset failed: %v, using by default utf-8", name, err)
  36. cs = "utf-8"
  37. }
  38. ctx.Resp.Header().Set("Content-Type", "text/plain; charset="+strings.ToLower(cs))
  39. } else if base.IsImageFile(buf) || base.IsPDFFile(buf) {
  40. ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename="%s"`, name))
  41. ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition")
  42. } else {
  43. ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, name))
  44. ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition")
  45. }
  46. _, err = ctx.Resp.Write(buf)
  47. if err != nil {
  48. return err
  49. }
  50. _, err = io.Copy(ctx.Resp, reader)
  51. return err
  52. }
  53. // ServeBlob download a git.Blob
  54. func ServeBlob(ctx *context.Context, blob *git.Blob) error {
  55. dataRc, err := blob.DataAsync()
  56. if err != nil {
  57. return err
  58. }
  59. defer func() {
  60. if err = dataRc.Close(); err != nil {
  61. log.Error("ServeBlob: Close: %v", err)
  62. }
  63. }()
  64. return ServeData(ctx, ctx.Repo.TreePath, dataRc)
  65. }
  66. // ServeBlobOrLFS download a git.Blob redirecting to LFS if necessary
  67. func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error {
  68. dataRc, err := blob.DataAsync()
  69. if err != nil {
  70. return err
  71. }
  72. defer func() {
  73. if err = dataRc.Close(); err != nil {
  74. log.Error("ServeBlobOrLFS: Close: %v", err)
  75. }
  76. }()
  77. if meta, _ := lfs.ReadPointerFile(dataRc); meta != nil {
  78. meta, _ = ctx.Repo.Repository.GetLFSMetaObjectByOid(meta.Oid)
  79. if meta == nil {
  80. return ServeBlob(ctx, blob)
  81. }
  82. lfsDataRc, err := lfs.ReadMetaObject(meta)
  83. if err != nil {
  84. return err
  85. }
  86. defer func() {
  87. if err = lfsDataRc.Close(); err != nil {
  88. log.Error("ServeBlobOrLFS: Close: %v", err)
  89. }
  90. }()
  91. return ServeData(ctx, ctx.Repo.TreePath, lfsDataRc)
  92. }
  93. return ServeBlob(ctx, blob)
  94. }
  95. // SingleDownload download a file by repos path
  96. func SingleDownload(ctx *context.Context) {
  97. blob, err := ctx.Repo.Commit.GetBlobByPath(ctx.Repo.TreePath)
  98. if err != nil {
  99. if git.IsErrNotExist(err) {
  100. ctx.NotFound("GetBlobByPath", nil)
  101. } else {
  102. ctx.ServerError("GetBlobByPath", err)
  103. }
  104. return
  105. }
  106. if err = ServeBlob(ctx, blob); err != nil {
  107. ctx.ServerError("ServeBlob", err)
  108. }
  109. }
  110. // SingleDownloadOrLFS download a file by repos path redirecting to LFS if necessary
  111. func SingleDownloadOrLFS(ctx *context.Context) {
  112. blob, err := ctx.Repo.Commit.GetBlobByPath(ctx.Repo.TreePath)
  113. if err != nil {
  114. if git.IsErrNotExist(err) {
  115. ctx.NotFound("GetBlobByPath", nil)
  116. } else {
  117. ctx.ServerError("GetBlobByPath", err)
  118. }
  119. return
  120. }
  121. if err = ServeBlobOrLFS(ctx, blob); err != nil {
  122. ctx.ServerError("ServeBlobOrLFS", err)
  123. }
  124. }
  125. // DownloadByID download a file by sha1 ID
  126. func DownloadByID(ctx *context.Context) {
  127. blob, err := ctx.Repo.GitRepo.GetBlob(ctx.Params("sha"))
  128. if err != nil {
  129. if git.IsErrNotExist(err) {
  130. ctx.NotFound("GetBlob", nil)
  131. } else {
  132. ctx.ServerError("GetBlob", err)
  133. }
  134. return
  135. }
  136. if err = ServeBlob(ctx, blob); err != nil {
  137. ctx.ServerError("ServeBlob", err)
  138. }
  139. }
  140. // DownloadByIDOrLFS download a file by sha1 ID taking account of LFS
  141. func DownloadByIDOrLFS(ctx *context.Context) {
  142. blob, err := ctx.Repo.GitRepo.GetBlob(ctx.Params("sha"))
  143. if err != nil {
  144. if git.IsErrNotExist(err) {
  145. ctx.NotFound("GetBlob", nil)
  146. } else {
  147. ctx.ServerError("GetBlob", err)
  148. }
  149. return
  150. }
  151. if err = ServeBlobOrLFS(ctx, blob); err != nil {
  152. ctx.ServerError("ServeBlob", err)
  153. }
  154. }