You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

342 lines
12 KiB

Graceful Queues: Issue Indexing and Tasks (#9363) * Queue: Add generic graceful queues with settings * Queue & Setting: Add worker pool implementation * Queue: Add worker settings * Queue: Make resizing worker pools * Queue: Add name variable to queues * Queue: Add monitoring * Queue: Improve logging * Issues: Gracefulise the issues indexer Remove the old now unused specific queues * Task: Move to generic queue and gracefulise * Issues: Standardise the issues indexer queue settings * Fix test * Queue: Allow Redis to connect to unix * Prevent deadlock during early shutdown of issue indexer * Add MaxWorker settings to queues * Merge branch 'master' into graceful-queues * Update modules/indexer/issues/indexer.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/indexer/issues/indexer.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/queue/queue_channel.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/queue/queue_disk.go * Update modules/queue/queue_disk_channel.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Rename queue.Description to queue.ManagedQueue as per @guillep2k * Cancel pool workers when removed * Remove dependency on queue from setting * Update modules/queue/queue_redis.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * As per @guillep2k add mutex locks on shutdown/terminate * move unlocking out of setInternal * Add warning if number of workers < 0 * Small changes as per @guillep2k * No redis host specified not found * Clean up documentation for queues * Update docs/content/doc/advanced/config-cheat-sheet.en-us.md * Update modules/indexer/issues/indexer_test.go * Ensure that persistable channel queue is added to manager * Rename QUEUE_NAME REDIS_QUEUE_NAME * Revert "Rename QUEUE_NAME REDIS_QUEUE_NAME" This reverts commit 1f83b4fc9b9dabda186257b38c265fe7012f90df. Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: techknowlogick <matti@mdranta.net> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
4 years ago
Graceful Queues: Issue Indexing and Tasks (#9363) * Queue: Add generic graceful queues with settings * Queue & Setting: Add worker pool implementation * Queue: Add worker settings * Queue: Make resizing worker pools * Queue: Add name variable to queues * Queue: Add monitoring * Queue: Improve logging * Issues: Gracefulise the issues indexer Remove the old now unused specific queues * Task: Move to generic queue and gracefulise * Issues: Standardise the issues indexer queue settings * Fix test * Queue: Allow Redis to connect to unix * Prevent deadlock during early shutdown of issue indexer * Add MaxWorker settings to queues * Merge branch 'master' into graceful-queues * Update modules/indexer/issues/indexer.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/indexer/issues/indexer.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/queue/queue_channel.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/queue/queue_disk.go * Update modules/queue/queue_disk_channel.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Rename queue.Description to queue.ManagedQueue as per @guillep2k * Cancel pool workers when removed * Remove dependency on queue from setting * Update modules/queue/queue_redis.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * As per @guillep2k add mutex locks on shutdown/terminate * move unlocking out of setInternal * Add warning if number of workers < 0 * Small changes as per @guillep2k * No redis host specified not found * Clean up documentation for queues * Update docs/content/doc/advanced/config-cheat-sheet.en-us.md * Update modules/indexer/issues/indexer_test.go * Ensure that persistable channel queue is added to manager * Rename QUEUE_NAME REDIS_QUEUE_NAME * Revert "Rename QUEUE_NAME REDIS_QUEUE_NAME" This reverts commit 1f83b4fc9b9dabda186257b38c265fe7012f90df. Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: techknowlogick <matti@mdranta.net> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
4 years ago
Graceful Queues: Issue Indexing and Tasks (#9363) * Queue: Add generic graceful queues with settings * Queue & Setting: Add worker pool implementation * Queue: Add worker settings * Queue: Make resizing worker pools * Queue: Add name variable to queues * Queue: Add monitoring * Queue: Improve logging * Issues: Gracefulise the issues indexer Remove the old now unused specific queues * Task: Move to generic queue and gracefulise * Issues: Standardise the issues indexer queue settings * Fix test * Queue: Allow Redis to connect to unix * Prevent deadlock during early shutdown of issue indexer * Add MaxWorker settings to queues * Merge branch 'master' into graceful-queues * Update modules/indexer/issues/indexer.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/indexer/issues/indexer.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/queue/queue_channel.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/queue/queue_disk.go * Update modules/queue/queue_disk_channel.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Rename queue.Description to queue.ManagedQueue as per @guillep2k * Cancel pool workers when removed * Remove dependency on queue from setting * Update modules/queue/queue_redis.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * As per @guillep2k add mutex locks on shutdown/terminate * move unlocking out of setInternal * Add warning if number of workers < 0 * Small changes as per @guillep2k * No redis host specified not found * Clean up documentation for queues * Update docs/content/doc/advanced/config-cheat-sheet.en-us.md * Update modules/indexer/issues/indexer_test.go * Ensure that persistable channel queue is added to manager * Rename QUEUE_NAME REDIS_QUEUE_NAME * Revert "Rename QUEUE_NAME REDIS_QUEUE_NAME" This reverts commit 1f83b4fc9b9dabda186257b38c265fe7012f90df. Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: techknowlogick <matti@mdranta.net> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
4 years ago
  1. // Copyright 2017 The Gitea Authors. All rights reserved.
  2. // Use of this source code is governed by a MIT-style
  3. // license that can be found in the LICENSE file.
  4. package integrations
  5. import (
  6. "fmt"
  7. "net/http"
  8. "path"
  9. "strconv"
  10. "strings"
  11. "testing"
  12. "time"
  13. "code.gitea.io/gitea/models"
  14. "code.gitea.io/gitea/modules/indexer/issues"
  15. "code.gitea.io/gitea/modules/references"
  16. "code.gitea.io/gitea/modules/setting"
  17. "code.gitea.io/gitea/modules/test"
  18. "github.com/PuerkitoBio/goquery"
  19. "github.com/stretchr/testify/assert"
  20. )
  21. func getIssuesSelection(t testing.TB, htmlDoc *HTMLDoc) *goquery.Selection {
  22. issueList := htmlDoc.doc.Find(".issue.list")
  23. assert.EqualValues(t, 1, issueList.Length())
  24. return issueList.Find("li").Find(".title")
  25. }
  26. func getIssue(t *testing.T, repoID int64, issueSelection *goquery.Selection) *models.Issue {
  27. href, exists := issueSelection.Attr("href")
  28. assert.True(t, exists)
  29. indexStr := href[strings.LastIndexByte(href, '/')+1:]
  30. index, err := strconv.Atoi(indexStr)
  31. assert.NoError(t, err, "Invalid issue href: %s", href)
  32. return models.AssertExistsAndLoadBean(t, &models.Issue{RepoID: repoID, Index: int64(index)}).(*models.Issue)
  33. }
  34. func assertMatch(t testing.TB, issue *models.Issue, keyword string) {
  35. matches := strings.Contains(strings.ToLower(issue.Title), keyword) ||
  36. strings.Contains(strings.ToLower(issue.Content), keyword)
  37. for _, comment := range issue.Comments {
  38. matches = matches || strings.Contains(
  39. strings.ToLower(comment.Content),
  40. keyword,
  41. )
  42. }
  43. assert.True(t, matches)
  44. }
  45. func TestNoLoginViewIssues(t *testing.T) {
  46. defer prepareTestEnv(t)()
  47. req := NewRequest(t, "GET", "/user2/repo1/issues")
  48. MakeRequest(t, req, http.StatusOK)
  49. }
  50. func TestViewIssuesSortByType(t *testing.T) {
  51. defer prepareTestEnv(t)()
  52. user := models.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User)
  53. repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
  54. session := loginUser(t, user.Name)
  55. req := NewRequest(t, "GET", repo.RelLink()+"/issues?type=created_by")
  56. resp := session.MakeRequest(t, req, http.StatusOK)
  57. htmlDoc := NewHTMLParser(t, resp.Body)
  58. issuesSelection := getIssuesSelection(t, htmlDoc)
  59. expectedNumIssues := models.GetCount(t,
  60. &models.Issue{RepoID: repo.ID, PosterID: user.ID},
  61. models.Cond("is_closed=?", false),
  62. models.Cond("is_pull=?", false),
  63. )
  64. if expectedNumIssues > setting.UI.IssuePagingNum {
  65. expectedNumIssues = setting.UI.IssuePagingNum
  66. }
  67. assert.EqualValues(t, expectedNumIssues, issuesSelection.Length())
  68. issuesSelection.Each(func(_ int, selection *goquery.Selection) {
  69. issue := getIssue(t, repo.ID, selection)
  70. assert.EqualValues(t, user.ID, issue.PosterID)
  71. })
  72. }
  73. func TestViewIssuesKeyword(t *testing.T) {
  74. defer prepareTestEnv(t)()
  75. repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
  76. issue := models.AssertExistsAndLoadBean(t, &models.Issue{
  77. RepoID: repo.ID,
  78. Index: 1,
  79. }).(*models.Issue)
  80. issues.UpdateIssueIndexer(issue)
  81. time.Sleep(time.Second * 1)
  82. const keyword = "first"
  83. req := NewRequestf(t, "GET", "%s/issues?q=%s", repo.RelLink(), keyword)
  84. resp := MakeRequest(t, req, http.StatusOK)
  85. htmlDoc := NewHTMLParser(t, resp.Body)
  86. issuesSelection := getIssuesSelection(t, htmlDoc)
  87. assert.EqualValues(t, 1, issuesSelection.Length())
  88. issuesSelection.Each(func(_ int, selection *goquery.Selection) {
  89. issue := getIssue(t, repo.ID, selection)
  90. assert.False(t, issue.IsClosed)
  91. assert.False(t, issue.IsPull)
  92. assertMatch(t, issue, keyword)
  93. })
  94. }
  95. func TestNoLoginViewIssue(t *testing.T) {
  96. defer prepareTestEnv(t)()
  97. req := NewRequest(t, "GET", "/user2/repo1/issues/1")
  98. MakeRequest(t, req, http.StatusOK)
  99. }
  100. func testNewIssue(t *testing.T, session *TestSession, user, repo, title, content string) string {
  101. req := NewRequest(t, "GET", path.Join(user, repo, "issues", "new"))
  102. resp := session.MakeRequest(t, req, http.StatusOK)
  103. htmlDoc := NewHTMLParser(t, resp.Body)
  104. link, exists := htmlDoc.doc.Find("form.ui.form").Attr("action")
  105. assert.True(t, exists, "The template has changed")
  106. req = NewRequestWithValues(t, "POST", link, map[string]string{
  107. "_csrf": htmlDoc.GetCSRF(),
  108. "title": title,
  109. "content": content,
  110. })
  111. resp = session.MakeRequest(t, req, http.StatusFound)
  112. issueURL := test.RedirectURL(resp)
  113. req = NewRequest(t, "GET", issueURL)
  114. resp = session.MakeRequest(t, req, http.StatusOK)
  115. htmlDoc = NewHTMLParser(t, resp.Body)
  116. val := htmlDoc.doc.Find("#issue-title").Text()
  117. assert.Equal(t, title, val)
  118. val = htmlDoc.doc.Find(".comment .render-content p").First().Text()
  119. assert.Equal(t, content, val)
  120. return issueURL
  121. }
  122. func testIssueAddComment(t *testing.T, session *TestSession, issueURL, content, status string) int64 {
  123. req := NewRequest(t, "GET", issueURL)
  124. resp := session.MakeRequest(t, req, http.StatusOK)
  125. htmlDoc := NewHTMLParser(t, resp.Body)
  126. link, exists := htmlDoc.doc.Find("#comment-form").Attr("action")
  127. assert.True(t, exists, "The template has changed")
  128. commentCount := htmlDoc.doc.Find(".comment-list .comment .render-content").Length()
  129. req = NewRequestWithValues(t, "POST", link, map[string]string{
  130. "_csrf": htmlDoc.GetCSRF(),
  131. "content": content,
  132. "status": status,
  133. })
  134. resp = session.MakeRequest(t, req, http.StatusFound)
  135. req = NewRequest(t, "GET", test.RedirectURL(resp))
  136. resp = session.MakeRequest(t, req, http.StatusOK)
  137. htmlDoc = NewHTMLParser(t, resp.Body)
  138. val := htmlDoc.doc.Find(".comment-list .comment .render-content p").Eq(commentCount).Text()
  139. assert.Equal(t, content, val)
  140. idAttr, has := htmlDoc.doc.Find(".comment-list .comment").Eq(commentCount).Attr("id")
  141. idStr := idAttr[strings.LastIndexByte(idAttr, '-')+1:]
  142. assert.True(t, has)
  143. id, err := strconv.Atoi(idStr)
  144. assert.NoError(t, err)
  145. return int64(id)
  146. }
  147. func TestNewIssue(t *testing.T) {
  148. defer prepareTestEnv(t)()
  149. session := loginUser(t, "user2")
  150. testNewIssue(t, session, "user2", "repo1", "Title", "Description")
  151. }
  152. func TestIssueCommentClose(t *testing.T) {
  153. defer prepareTestEnv(t)()
  154. session := loginUser(t, "user2")
  155. issueURL := testNewIssue(t, session, "user2", "repo1", "Title", "Description")
  156. testIssueAddComment(t, session, issueURL, "Test comment 1", "")
  157. testIssueAddComment(t, session, issueURL, "Test comment 2", "")
  158. testIssueAddComment(t, session, issueURL, "Test comment 3", "close")
  159. // Validate that issue content has not been updated
  160. req := NewRequest(t, "GET", issueURL)
  161. resp := session.MakeRequest(t, req, http.StatusOK)
  162. htmlDoc := NewHTMLParser(t, resp.Body)
  163. val := htmlDoc.doc.Find(".comment-list .comment .render-content p").First().Text()
  164. assert.Equal(t, "Description", val)
  165. }
  166. func TestIssueReaction(t *testing.T) {
  167. defer prepareTestEnv(t)()
  168. session := loginUser(t, "user2")
  169. issueURL := testNewIssue(t, session, "user2", "repo1", "Title", "Description")
  170. req := NewRequest(t, "GET", issueURL)
  171. resp := session.MakeRequest(t, req, http.StatusOK)
  172. htmlDoc := NewHTMLParser(t, resp.Body)
  173. req = NewRequestWithValues(t, "POST", path.Join(issueURL, "/reactions/react"), map[string]string{
  174. "_csrf": htmlDoc.GetCSRF(),
  175. "content": "8ball",
  176. })
  177. session.MakeRequest(t, req, http.StatusInternalServerError)
  178. req = NewRequestWithValues(t, "POST", path.Join(issueURL, "/reactions/react"), map[string]string{
  179. "_csrf": htmlDoc.GetCSRF(),
  180. "content": "eyes",
  181. })
  182. session.MakeRequest(t, req, http.StatusOK)
  183. req = NewRequestWithValues(t, "POST", path.Join(issueURL, "/reactions/unreact"), map[string]string{
  184. "_csrf": htmlDoc.GetCSRF(),
  185. "content": "eyes",
  186. })
  187. session.MakeRequest(t, req, http.StatusOK)
  188. }
  189. func TestIssueCrossReference(t *testing.T) {
  190. defer prepareTestEnv(t)()
  191. // Issue that will be referenced
  192. _, issueBase := testIssueWithBean(t, "user2", 1, "Title", "Description")
  193. // Ref from issue title
  194. issueRefURL, issueRef := testIssueWithBean(t, "user2", 1, fmt.Sprintf("Title ref #%d", issueBase.Index), "Description")
  195. models.AssertExistsAndLoadBean(t, &models.Comment{
  196. IssueID: issueBase.ID,
  197. RefRepoID: 1,
  198. RefIssueID: issueRef.ID,
  199. RefCommentID: 0,
  200. RefIsPull: false,
  201. RefAction: references.XRefActionNone})
  202. // Edit title, neuter ref
  203. testIssueChangeInfo(t, "user2", issueRefURL, "title", "Title no ref")
  204. models.AssertExistsAndLoadBean(t, &models.Comment{
  205. IssueID: issueBase.ID,
  206. RefRepoID: 1,
  207. RefIssueID: issueRef.ID,
  208. RefCommentID: 0,
  209. RefIsPull: false,
  210. RefAction: references.XRefActionNeutered})
  211. // Ref from issue content
  212. issueRefURL, issueRef = testIssueWithBean(t, "user2", 1, "TitleXRef", fmt.Sprintf("Description ref #%d", issueBase.Index))
  213. models.AssertExistsAndLoadBean(t, &models.Comment{
  214. IssueID: issueBase.ID,
  215. RefRepoID: 1,
  216. RefIssueID: issueRef.ID,
  217. RefCommentID: 0,
  218. RefIsPull: false,
  219. RefAction: references.XRefActionNone})
  220. // Edit content, neuter ref
  221. testIssueChangeInfo(t, "user2", issueRefURL, "content", "Description no ref")
  222. models.AssertExistsAndLoadBean(t, &models.Comment{
  223. IssueID: issueBase.ID,
  224. RefRepoID: 1,
  225. RefIssueID: issueRef.ID,
  226. RefCommentID: 0,
  227. RefIsPull: false,
  228. RefAction: references.XRefActionNeutered})
  229. // Ref from a comment
  230. session := loginUser(t, "user2")
  231. commentID := testIssueAddComment(t, session, issueRefURL, fmt.Sprintf("Adding ref from comment #%d", issueBase.Index), "")
  232. comment := &models.Comment{
  233. IssueID: issueBase.ID,
  234. RefRepoID: 1,
  235. RefIssueID: issueRef.ID,
  236. RefCommentID: commentID,
  237. RefIsPull: false,
  238. RefAction: references.XRefActionNone}
  239. models.AssertExistsAndLoadBean(t, comment)
  240. // Ref from a different repository
  241. issueRefURL, issueRef = testIssueWithBean(t, "user12", 10, "TitleXRef", fmt.Sprintf("Description ref user2/repo1#%d", issueBase.Index))
  242. models.AssertExistsAndLoadBean(t, &models.Comment{
  243. IssueID: issueBase.ID,
  244. RefRepoID: 10,
  245. RefIssueID: issueRef.ID,
  246. RefCommentID: 0,
  247. RefIsPull: false,
  248. RefAction: references.XRefActionNone})
  249. }
  250. func testIssueWithBean(t *testing.T, user string, repoID int64, title, content string) (string, *models.Issue) {
  251. session := loginUser(t, user)
  252. issueURL := testNewIssue(t, session, user, fmt.Sprintf("repo%d", repoID), title, content)
  253. indexStr := issueURL[strings.LastIndexByte(issueURL, '/')+1:]
  254. index, err := strconv.Atoi(indexStr)
  255. assert.NoError(t, err, "Invalid issue href: %s", issueURL)
  256. issue := &models.Issue{RepoID: repoID, Index: int64(index)}
  257. models.AssertExistsAndLoadBean(t, issue)
  258. return issueURL, issue
  259. }
  260. func testIssueChangeInfo(t *testing.T, user, issueURL, info string, value string) {
  261. session := loginUser(t, user)
  262. req := NewRequest(t, "GET", issueURL)
  263. resp := session.MakeRequest(t, req, http.StatusOK)
  264. htmlDoc := NewHTMLParser(t, resp.Body)
  265. req = NewRequestWithValues(t, "POST", path.Join(issueURL, info), map[string]string{
  266. "_csrf": htmlDoc.GetCSRF(),
  267. info: value,
  268. })
  269. _ = session.MakeRequest(t, req, http.StatusOK)
  270. }
  271. func TestIssueRedirect(t *testing.T) {
  272. defer prepareTestEnv(t)()
  273. session := loginUser(t, "user2")
  274. // Test external tracker where style not set (shall default numeric)
  275. req := NewRequest(t, "GET", path.Join("org26", "repo_external_tracker", "issues", "1"))
  276. resp := session.MakeRequest(t, req, http.StatusFound)
  277. assert.Equal(t, "https://tracker.com/org26/repo_external_tracker/issues/1", test.RedirectURL(resp))
  278. // Test external tracker with numeric style
  279. req = NewRequest(t, "GET", path.Join("org26", "repo_external_tracker_numeric", "issues", "1"))
  280. resp = session.MakeRequest(t, req, http.StatusFound)
  281. assert.Equal(t, "https://tracker.com/org26/repo_external_tracker_numeric/issues/1", test.RedirectURL(resp))
  282. // Test external tracker with alphanumeric style (for a pull request)
  283. req = NewRequest(t, "GET", path.Join("org26", "repo_external_tracker_alpha", "issues", "1"))
  284. resp = session.MakeRequest(t, req, http.StatusFound)
  285. assert.Equal(t, "/"+path.Join("org26", "repo_external_tracker_alpha", "pulls", "1"), test.RedirectURL(resp))
  286. }