2022-09-02 15:18:23 -04:00
|
|
|
// Copyright 2017 The Gitea Authors. All rights reserved.
|
2022-11-27 13:20:29 -05:00
|
|
|
// SPDX-License-Identifier: MIT
|
2022-09-02 15:18:23 -04:00
|
|
|
|
Enable forbidigo linter (#24278)
Enable [forbidigo](https://github.com/ashanbrown/forbidigo) linter which
forbids print statements. Will check how to integrate this with the
smallest impact possible, so a few `nolint` comments will likely be
required. Plan is to just go through the issues and either:
- Remove the print if it is nonsensical
- Add a `//nolint` directive if it makes sense
I don't plan on investigating the individual issues any further.
<details>
<summary>Initial Lint Results</summary>
```
modules/log/event.go:348:6: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println(err)
^
modules/log/event.go:382:6: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println(err)
^
modules/queue/unique_queue_disk_channel_test.go:20:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("TempDir %s\n", tmpDir)
^
contrib/backport/backport.go:168:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("* Backporting %s to %s as %s\n", pr, localReleaseBranch, backportBranch)
^
contrib/backport/backport.go:216:4: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("* Navigate to %s to open PR\n", url)
^
contrib/backport/backport.go:223:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("* `xdg-open %s`\n", url)
^
contrib/backport/backport.go:233:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("* `git push -u %s %s`\n", remote, backportBranch)
^
contrib/backport/backport.go:243:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("* Amending commit to prepend `Backport #%s` to body\n", pr)
^
contrib/backport/backport.go:272:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println("* Attempting git cherry-pick --continue")
^
contrib/backport/backport.go:281:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("* Attempting git cherry-pick %s\n", sha)
^
contrib/backport/backport.go:297:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("* Current branch is %s\n", currentBranch)
^
contrib/backport/backport.go:299:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("* Current branch is %s - not checking out\n", currentBranch)
^
contrib/backport/backport.go:304:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("* Branch %s already exists. Checking it out...\n", backportBranch)
^
contrib/backport/backport.go:308:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("* `git checkout -b %s %s`\n", backportBranch, releaseBranch)
^
contrib/backport/backport.go:313:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("* `git fetch %s main`\n", remote)
^
contrib/backport/backport.go:316:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println(string(out))
^
contrib/backport/backport.go:319:2: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println(string(out))
^
contrib/backport/backport.go:321:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("* `git fetch %s %s`\n", remote, releaseBranch)
^
contrib/backport/backport.go:324:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println(string(out))
^
contrib/backport/backport.go:327:2: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println(string(out))
^
models/unittest/fixtures.go:50:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println("Unsupported RDBMS for integration tests")
^
models/unittest/fixtures.go:89:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("LoadFixtures failed after retries: %v\n", err)
^
models/unittest/fixtures.go:110:4: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("Failed to generate sequence update: %v\n", err)
^
models/unittest/fixtures.go:117:6: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("Failed to update sequence: %s Error: %v\n", value, err)
^
models/migrations/base/tests.go:118:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println("Environment variable $GITEA_ROOT not set")
^
models/migrations/base/tests.go:127:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("Could not find gitea binary at %s\n", setting.AppPath)
^
models/migrations/base/tests.go:134:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("Environment variable $GITEA_CONF not set - defaulting to %s\n", giteaConf)
^
models/migrations/base/tests.go:145:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("Unable to create temporary data path %v\n", err)
^
models/migrations/base/tests.go:154:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("Unable to InitFull: %v\n", err)
^
models/migrations/v1_11/v112.go:34:5: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("Error: %v", err)
^
contrib/fixtures/fixture_generation.go:36:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("CreateTestEngine: %+v", err)
^
contrib/fixtures/fixture_generation.go:40:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("PrepareTestDatabase: %+v\n", err)
^
contrib/fixtures/fixture_generation.go:46:5: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("generate '%s': %+v\n", r, err)
^
contrib/fixtures/fixture_generation.go:53:5: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("generate '%s': %+v\n", g.name, err)
^
contrib/fixtures/fixture_generation.go:71:4: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("%s created.\n", path)
^
services/gitdiff/gitdiff_test.go:543:2: use of `println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
println(result)
^
services/gitdiff/gitdiff_test.go:560:2: use of `println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
println(result)
^
services/gitdiff/gitdiff_test.go:577:2: use of `println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
println(result)
^
modules/web/routing/logger_manager.go:34:2: use of `print` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
print Printer
^
modules/doctor/paths.go:109:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("Warning: can't remove temporary file: '%s'\n", tmpFile.Name())
^
tests/test_utils.go:33:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf(format+"\n", args...)
^
tests/test_utils.go:61:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("Environment variable $GITEA_CONF not set, use default: %s\n", giteaConf)
^
cmd/actions.go:54:9: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
_, _ = fmt.Printf("%s\n", respText)
^
cmd/admin_user_change_password.go:74:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("%s's password has been successfully updated!\n", user.Name)
^
cmd/admin_user_create.go:109:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("generated random password is '%s'\n", password)
^
cmd/admin_user_create.go:164:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("Access token was successfully created... %s\n", t.Token)
^
cmd/admin_user_create.go:167:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("New user '%s' has been successfully created!\n", username)
^
cmd/admin_user_generate_access_token.go:74:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("%s\n", t.Token)
^
cmd/admin_user_generate_access_token.go:76:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("Access token was successfully created: %s\n", t.Token)
^
cmd/admin_user_must_change_password.go:56:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("Updated %d users setting MustChangePassword to %t\n", n, mustChangePassword)
^
cmd/convert.go:44:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println("Converted successfully, please confirm your database's character set is now utf8mb4")
^
cmd/convert.go:50:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println("Converted successfully, please confirm your database's all columns character is NVARCHAR now")
^
cmd/convert.go:52:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println("This command can only be used with a MySQL or MSSQL database")
^
cmd/doctor.go:104:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println(err)
^
cmd/doctor.go:105:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println("Check if you are using the right config file. You can use a --config directive to specify one.")
^
cmd/doctor.go:243:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println(err)
^
cmd/embedded.go:154:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println(a.path)
^
cmd/embedded.go:198:3: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println("Using app.ini at", setting.CustomConf)
^
cmd/embedded.go:217:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("Extracting to %s:\n", destdir)
^
cmd/embedded.go:253:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("%s already exists; skipped.\n", dest)
^
cmd/embedded.go:275:2: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println(dest)
^
cmd/generate.go:63:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("%s", internalToken)
^
cmd/generate.go:66:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("\n")
^
cmd/generate.go:78:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("%s", JWTSecretBase64)
^
cmd/generate.go:81:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("\n")
^
cmd/generate.go:93:2: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("%s", secretKey)
^
cmd/generate.go:96:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("\n")
^
cmd/keys.go:74:2: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println(strings.TrimSpace(authorizedString))
^
cmd/mailer.go:32:4: use of `fmt.Print` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Print("warning: Content is empty")
^
cmd/mailer.go:35:3: use of `fmt.Print` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Print("Proceed with sending email? [Y/n] ")
^
cmd/mailer.go:40:4: use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Println("The mail was not sent")
^
cmd/mailer.go:49:9: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
_, _ = fmt.Printf("Sent %s email(s) to all users\n", respText)
^
cmd/serv.go:147:3: use of `println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
println("Gitea: SSH has been disabled")
^
cmd/serv.go:153:4: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("error showing subcommand help: %v\n", err)
^
cmd/serv.go:175:4: use of `println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
println("Hi there! You've successfully authenticated with the deploy key named " + key.Name + ", but Gitea does not provide shell access.")
^
cmd/serv.go:177:4: use of `println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
println("Hi there! You've successfully authenticated with the principal " + key.Content + ", but Gitea does not provide shell access.")
^
cmd/serv.go:179:4: use of `println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
println("Hi there, " + user.Name + "! You've successfully authenticated with the key named " + key.Name + ", but Gitea does not provide shell access.")
^
cmd/serv.go:181:3: use of `println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
println("If this is unexpected, please log in with password and setup Gitea under another user.")
^
cmd/serv.go:196:5: use of `fmt.Print` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Print(`{"type":"gitea","version":1}`)
^
tests/e2e/e2e_test.go:54:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("Error initializing test database: %v\n", err)
^
tests/e2e/e2e_test.go:63:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("util.RemoveAll: %v\n", err)
^
tests/e2e/e2e_test.go:67:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("Unable to remove repo indexer: %v\n", err)
^
tests/e2e/e2e_test.go:109:6: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("%v", stdout.String())
^
tests/e2e/e2e_test.go:110:6: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("%v", stderr.String())
^
tests/e2e/e2e_test.go:113:6: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("%v", stdout.String())
^
tests/integration/integration_test.go:124:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("Error initializing test database: %v\n", err)
^
tests/integration/integration_test.go:135:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("util.RemoveAll: %v\n", err)
^
tests/integration/integration_test.go:139:3: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("Unable to remove repo indexer: %v\n", err)
^
tests/integration/repo_test.go:357:4: use of `fmt.Printf` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
fmt.Printf("%s", resp.Body)
^
```
</details>
---------
Co-authored-by: Giteabot <teabot@gitea.io>
2023-04-24 05:50:58 -04:00
|
|
|
//nolint:forbidigo
|
2022-09-02 15:18:23 -04:00
|
|
|
package tests
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"context"
|
|
|
|
|
"database/sql"
|
|
|
|
|
"fmt"
|
|
|
|
|
"os"
|
|
|
|
|
"path"
|
|
|
|
|
"path/filepath"
|
|
|
|
|
"testing"
|
|
|
|
|
|
2023-02-23 09:11:56 -05:00
|
|
|
"code.gitea.io/gitea/models/db"
|
|
|
|
|
packages_model "code.gitea.io/gitea/models/packages"
|
2022-09-02 15:18:23 -04:00
|
|
|
"code.gitea.io/gitea/models/unittest"
|
|
|
|
|
"code.gitea.io/gitea/modules/base"
|
|
|
|
|
"code.gitea.io/gitea/modules/git"
|
|
|
|
|
"code.gitea.io/gitea/modules/graceful"
|
|
|
|
|
"code.gitea.io/gitea/modules/log"
|
|
|
|
|
repo_module "code.gitea.io/gitea/modules/repository"
|
|
|
|
|
"code.gitea.io/gitea/modules/setting"
|
|
|
|
|
"code.gitea.io/gitea/modules/storage"
|
Rewrite queue (#24505)
# ⚠️ Breaking
Many deprecated queue config options are removed (actually, they should
have been removed in 1.18/1.19).
If you see the fatal message when starting Gitea: "Please update your
app.ini to remove deprecated config options", please follow the error
messages to remove these options from your app.ini.
Example:
```
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options
```
Many options in `[queue]` are are dropped, including:
`WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`,
`BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed
from app.ini.
# The problem
The old queue package has some legacy problems:
* complexity: I doubt few people could tell how it works.
* maintainability: Too many channels and mutex/cond are mixed together,
too many different structs/interfaces depends each other.
* stability: due to the complexity & maintainability, sometimes there
are strange bugs and difficult to debug, and some code doesn't have test
(indeed some code is difficult to test because a lot of things are mixed
together).
* general applicability: although it is called "queue", its behavior is
not a well-known queue.
* scalability: it doesn't seem easy to make it work with a cluster
without breaking its behaviors.
It came from some very old code to "avoid breaking", however, its
technical debt is too heavy now. It's a good time to introduce a better
"queue" package.
# The new queue package
It keeps using old config and concept as much as possible.
* It only contains two major kinds of concepts:
* The "base queue": channel, levelqueue, redis
* They have the same abstraction, the same interface, and they are
tested by the same testing code.
* The "WokerPoolQueue", it uses the "base queue" to provide "worker
pool" function, calls the "handler" to process the data in the base
queue.
* The new code doesn't do "PushBack"
* Think about a queue with many workers, the "PushBack" can't guarantee
the order for re-queued unhandled items, so in new code it just does
"normal push"
* The new code doesn't do "pause/resume"
* The "pause/resume" was designed to handle some handler's failure: eg:
document indexer (elasticsearch) is down
* If a queue is paused for long time, either the producers blocks or the
new items are dropped.
* The new code doesn't do such "pause/resume" trick, it's not a common
queue's behavior and it doesn't help much.
* If there are unhandled items, the "push" function just blocks for a
few seconds and then re-queue them and retry.
* The new code doesn't do "worker booster"
* Gitea's queue's handlers are light functions, the cost is only the
go-routine, so it doesn't make sense to "boost" them.
* The new code only use "max worker number" to limit the concurrent
workers.
* The new "Push" never blocks forever
* Instead of creating more and more blocking goroutines, return an error
is more friendly to the server and to the end user.
There are more details in code comments: eg: the "Flush" problem, the
strange "code.index" hanging problem, the "immediate" queue problem.
Almost ready for review.
TODO:
* [x] add some necessary comments during review
* [x] add some more tests if necessary
* [x] update documents and config options
* [x] test max worker / active worker
* [x] re-run the CI tasks to see whether any test is flaky
* [x] improve the `handleOldLengthConfiguration` to provide more
friendly messages
* [x] fine tune default config values (eg: length?)
## Code coverage:
2023-05-08 07:49:59 -04:00
|
|
|
"code.gitea.io/gitea/modules/testlogger"
|
2022-09-02 15:18:23 -04:00
|
|
|
"code.gitea.io/gitea/modules/util"
|
|
|
|
|
"code.gitea.io/gitea/routers"
|
|
|
|
|
|
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
|
)
|
|
|
|
|
|
2023-04-19 09:40:42 -04:00
|
|
|
func exitf(format string, args ...interface{}) {
|
|
|
|
|
fmt.Printf(format+"\n", args...)
|
|
|
|
|
os.Exit(1)
|
|
|
|
|
}
|
|
|
|
|
|
2022-09-02 15:18:23 -04:00
|
|
|
func InitTest(requireGitea bool) {
|
|
|
|
|
giteaRoot := base.SetupGiteaRoot()
|
|
|
|
|
if giteaRoot == "" {
|
2023-04-19 09:40:42 -04:00
|
|
|
exitf("Environment variable $GITEA_ROOT not set")
|
2022-09-02 15:18:23 -04:00
|
|
|
}
|
2023-04-19 09:40:42 -04:00
|
|
|
setting.AppWorkPath = giteaRoot
|
2022-09-02 15:18:23 -04:00
|
|
|
if requireGitea {
|
|
|
|
|
giteaBinary := "gitea"
|
2023-04-19 09:40:42 -04:00
|
|
|
if setting.IsWindows {
|
2022-09-02 15:18:23 -04:00
|
|
|
giteaBinary += ".exe"
|
|
|
|
|
}
|
|
|
|
|
setting.AppPath = path.Join(giteaRoot, giteaBinary)
|
|
|
|
|
if _, err := os.Stat(setting.AppPath); err != nil {
|
2023-04-19 09:40:42 -04:00
|
|
|
exitf("Could not find gitea binary at %s", setting.AppPath)
|
2022-09-02 15:18:23 -04:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
giteaConf := os.Getenv("GITEA_CONF")
|
|
|
|
|
if giteaConf == "" {
|
2023-04-19 09:40:42 -04:00
|
|
|
// By default, use sqlite.ini for testing, then IDE like GoLand can start the test process with debugger.
|
|
|
|
|
// It's easier for developers to debug bugs step by step with a debugger.
|
|
|
|
|
// Notice: when doing "ssh push", Gitea executes sub processes, debugger won't work for the sub processes.
|
|
|
|
|
giteaConf = "tests/sqlite.ini"
|
|
|
|
|
_ = os.Setenv("GITEA_CONF", giteaConf)
|
|
|
|
|
fmt.Printf("Environment variable $GITEA_CONF not set, use default: %s\n", giteaConf)
|
|
|
|
|
if !setting.EnableSQLite3 {
|
Rewrite queue (#24505)
# ⚠️ Breaking
Many deprecated queue config options are removed (actually, they should
have been removed in 1.18/1.19).
If you see the fatal message when starting Gitea: "Please update your
app.ini to remove deprecated config options", please follow the error
messages to remove these options from your app.ini.
Example:
```
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options
```
Many options in `[queue]` are are dropped, including:
`WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`,
`BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed
from app.ini.
# The problem
The old queue package has some legacy problems:
* complexity: I doubt few people could tell how it works.
* maintainability: Too many channels and mutex/cond are mixed together,
too many different structs/interfaces depends each other.
* stability: due to the complexity & maintainability, sometimes there
are strange bugs and difficult to debug, and some code doesn't have test
(indeed some code is difficult to test because a lot of things are mixed
together).
* general applicability: although it is called "queue", its behavior is
not a well-known queue.
* scalability: it doesn't seem easy to make it work with a cluster
without breaking its behaviors.
It came from some very old code to "avoid breaking", however, its
technical debt is too heavy now. It's a good time to introduce a better
"queue" package.
# The new queue package
It keeps using old config and concept as much as possible.
* It only contains two major kinds of concepts:
* The "base queue": channel, levelqueue, redis
* They have the same abstraction, the same interface, and they are
tested by the same testing code.
* The "WokerPoolQueue", it uses the "base queue" to provide "worker
pool" function, calls the "handler" to process the data in the base
queue.
* The new code doesn't do "PushBack"
* Think about a queue with many workers, the "PushBack" can't guarantee
the order for re-queued unhandled items, so in new code it just does
"normal push"
* The new code doesn't do "pause/resume"
* The "pause/resume" was designed to handle some handler's failure: eg:
document indexer (elasticsearch) is down
* If a queue is paused for long time, either the producers blocks or the
new items are dropped.
* The new code doesn't do such "pause/resume" trick, it's not a common
queue's behavior and it doesn't help much.
* If there are unhandled items, the "push" function just blocks for a
few seconds and then re-queue them and retry.
* The new code doesn't do "worker booster"
* Gitea's queue's handlers are light functions, the cost is only the
go-routine, so it doesn't make sense to "boost" them.
* The new code only use "max worker number" to limit the concurrent
workers.
* The new "Push" never blocks forever
* Instead of creating more and more blocking goroutines, return an error
is more friendly to the server and to the end user.
There are more details in code comments: eg: the "Flush" problem, the
strange "code.index" hanging problem, the "immediate" queue problem.
Almost ready for review.
TODO:
* [x] add some necessary comments during review
* [x] add some more tests if necessary
* [x] update documents and config options
* [x] test max worker / active worker
* [x] re-run the CI tasks to see whether any test is flaky
* [x] improve the `handleOldLengthConfiguration` to provide more
friendly messages
* [x] fine tune default config values (eg: length?)
## Code coverage:
2023-05-08 07:49:59 -04:00
|
|
|
exitf(`sqlite3 requires: import _ "github.com/mattn/go-sqlite3" or -tags sqlite,sqlite_unlock_notify`)
|
2023-04-19 09:40:42 -04:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
setting.IsInTesting = true
|
|
|
|
|
|
|
|
|
|
if !path.IsAbs(giteaConf) {
|
2022-09-02 15:18:23 -04:00
|
|
|
setting.CustomConf = path.Join(giteaRoot, giteaConf)
|
|
|
|
|
} else {
|
|
|
|
|
setting.CustomConf = giteaConf
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
setting.SetCustomPathAndConf("", "", "")
|
2023-05-03 23:55:35 -04:00
|
|
|
unittest.InitSettings()
|
2022-09-02 15:18:23 -04:00
|
|
|
setting.Repository.DefaultBranch = "master" // many test code still assume that default branch is called "master"
|
|
|
|
|
_ = util.RemoveAll(repo_module.LocalCopyPath())
|
|
|
|
|
|
|
|
|
|
if err := git.InitFull(context.Background()); err != nil {
|
|
|
|
|
log.Fatal("git.InitOnceWithSync: %v", err)
|
|
|
|
|
}
|
|
|
|
|
|
2023-02-19 11:12:01 -05:00
|
|
|
setting.LoadDBSetting()
|
2022-09-02 15:18:23 -04:00
|
|
|
if err := storage.Init(); err != nil {
|
2023-04-19 09:40:42 -04:00
|
|
|
exitf("Init storage failed: %v", err)
|
2022-09-02 15:18:23 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
switch {
|
2023-03-07 05:51:06 -05:00
|
|
|
case setting.Database.Type.IsMySQL():
|
2022-09-02 15:18:23 -04:00
|
|
|
connType := "tcp"
|
|
|
|
|
if len(setting.Database.Host) > 0 && setting.Database.Host[0] == '/' { // looks like a unix socket
|
|
|
|
|
connType = "unix"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@%s(%s)/",
|
|
|
|
|
setting.Database.User, setting.Database.Passwd, connType, setting.Database.Host))
|
|
|
|
|
defer db.Close()
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Fatal("sql.Open: %v", err)
|
|
|
|
|
}
|
|
|
|
|
if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", setting.Database.Name)); err != nil {
|
|
|
|
|
log.Fatal("db.Exec: %v", err)
|
|
|
|
|
}
|
2023-03-07 05:51:06 -05:00
|
|
|
case setting.Database.Type.IsPostgreSQL():
|
2022-09-02 15:18:23 -04:00
|
|
|
var db *sql.DB
|
|
|
|
|
var err error
|
|
|
|
|
if setting.Database.Host[0] == '/' {
|
|
|
|
|
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@/%s?sslmode=%s&host=%s",
|
|
|
|
|
setting.Database.User, setting.Database.Passwd, setting.Database.Name, setting.Database.SSLMode, setting.Database.Host))
|
|
|
|
|
} else {
|
|
|
|
|
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s",
|
|
|
|
|
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
defer db.Close()
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Fatal("sql.Open: %v", err)
|
|
|
|
|
}
|
|
|
|
|
dbrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name))
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Fatal("db.Query: %v", err)
|
|
|
|
|
}
|
|
|
|
|
defer dbrows.Close()
|
|
|
|
|
|
|
|
|
|
if !dbrows.Next() {
|
|
|
|
|
if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name)); err != nil {
|
|
|
|
|
log.Fatal("db.Exec: CREATE DATABASE: %v", err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// Check if we need to setup a specific schema
|
|
|
|
|
if len(setting.Database.Schema) == 0 {
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
db.Close()
|
|
|
|
|
|
|
|
|
|
if setting.Database.Host[0] == '/' {
|
|
|
|
|
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@/%s?sslmode=%s&host=%s",
|
|
|
|
|
setting.Database.User, setting.Database.Passwd, setting.Database.Name, setting.Database.SSLMode, setting.Database.Host))
|
|
|
|
|
} else {
|
|
|
|
|
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s",
|
|
|
|
|
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode))
|
|
|
|
|
}
|
|
|
|
|
// This is a different db object; requires a different Close()
|
|
|
|
|
defer db.Close()
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Fatal("sql.Open: %v", err)
|
|
|
|
|
}
|
|
|
|
|
schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema))
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Fatal("db.Query: %v", err)
|
|
|
|
|
}
|
|
|
|
|
defer schrows.Close()
|
|
|
|
|
|
|
|
|
|
if !schrows.Next() {
|
|
|
|
|
// Create and setup a DB schema
|
|
|
|
|
if _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema)); err != nil {
|
|
|
|
|
log.Fatal("db.Exec: CREATE SCHEMA: %v", err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2023-03-07 05:51:06 -05:00
|
|
|
case setting.Database.Type.IsMSSQL():
|
2022-09-02 15:18:23 -04:00
|
|
|
host, port := setting.ParseMSSQLHostPort(setting.Database.Host)
|
|
|
|
|
db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
|
|
|
|
|
host, port, "master", setting.Database.User, setting.Database.Passwd))
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Fatal("sql.Open: %v", err)
|
|
|
|
|
}
|
|
|
|
|
if _, err := db.Exec(fmt.Sprintf("If(db_id(N'%s') IS NULL) BEGIN CREATE DATABASE %s; END;", setting.Database.Name, setting.Database.Name)); err != nil {
|
|
|
|
|
log.Fatal("db.Exec: %v", err)
|
|
|
|
|
}
|
|
|
|
|
defer db.Close()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
routers.GlobalInitInstalled(graceful.GetManager().HammerContext())
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func PrepareTestEnv(t testing.TB, skip ...int) func() {
|
|
|
|
|
t.Helper()
|
|
|
|
|
ourSkip := 2
|
|
|
|
|
if len(skip) > 0 {
|
|
|
|
|
ourSkip += skip[0]
|
|
|
|
|
}
|
|
|
|
|
deferFn := PrintCurrentTest(t, ourSkip)
|
2022-12-22 18:41:56 -05:00
|
|
|
|
|
|
|
|
// load database fixtures
|
2022-09-02 15:18:23 -04:00
|
|
|
assert.NoError(t, unittest.LoadFixtures())
|
2022-12-22 18:41:56 -05:00
|
|
|
|
|
|
|
|
// load git repo fixtures
|
2022-09-02 15:18:23 -04:00
|
|
|
assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
|
|
|
|
|
assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath))
|
|
|
|
|
ownerDirs, err := os.ReadDir(setting.RepoRootPath)
|
|
|
|
|
if err != nil {
|
|
|
|
|
assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
|
|
|
|
|
}
|
|
|
|
|
for _, ownerDir := range ownerDirs {
|
|
|
|
|
if !ownerDir.Type().IsDir() {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name()))
|
|
|
|
|
if err != nil {
|
|
|
|
|
assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
|
|
|
|
|
}
|
|
|
|
|
for _, repoDir := range repoDirs {
|
|
|
|
|
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755)
|
|
|
|
|
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755)
|
|
|
|
|
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755)
|
|
|
|
|
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2022-12-22 18:41:56 -05:00
|
|
|
// load LFS object fixtures
|
|
|
|
|
// (LFS storage can be on any of several backends, including remote servers, so we init it with the storage API)
|
|
|
|
|
lfsFixtures, err := storage.NewStorage("", storage.LocalStorageConfig{Path: path.Join(filepath.Dir(setting.AppPath), "tests/gitea-lfs-meta")})
|
|
|
|
|
assert.NoError(t, err)
|
|
|
|
|
assert.NoError(t, storage.Clean(storage.LFS))
|
2023-03-13 06:23:51 -04:00
|
|
|
assert.NoError(t, lfsFixtures.IterateObjects("", func(path string, _ storage.Object) error {
|
2022-12-22 18:41:56 -05:00
|
|
|
_, err := storage.Copy(storage.LFS, path, lfsFixtures, path)
|
|
|
|
|
return err
|
|
|
|
|
}))
|
|
|
|
|
|
2023-02-23 09:11:56 -05:00
|
|
|
// clear all package data
|
|
|
|
|
assert.NoError(t, db.TruncateBeans(db.DefaultContext,
|
|
|
|
|
&packages_model.Package{},
|
|
|
|
|
&packages_model.PackageVersion{},
|
|
|
|
|
&packages_model.PackageFile{},
|
|
|
|
|
&packages_model.PackageBlob{},
|
|
|
|
|
&packages_model.PackageProperty{},
|
|
|
|
|
&packages_model.PackageBlobUpload{},
|
|
|
|
|
&packages_model.PackageCleanupRule{},
|
|
|
|
|
))
|
|
|
|
|
assert.NoError(t, storage.Clean(storage.Packages))
|
|
|
|
|
|
2022-09-02 15:18:23 -04:00
|
|
|
return deferFn
|
|
|
|
|
}
|
|
|
|
|
|
Rewrite queue (#24505)
# ⚠️ Breaking
Many deprecated queue config options are removed (actually, they should
have been removed in 1.18/1.19).
If you see the fatal message when starting Gitea: "Please update your
app.ini to remove deprecated config options", please follow the error
messages to remove these options from your app.ini.
Example:
```
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options
```
Many options in `[queue]` are are dropped, including:
`WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`,
`BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed
from app.ini.
# The problem
The old queue package has some legacy problems:
* complexity: I doubt few people could tell how it works.
* maintainability: Too many channels and mutex/cond are mixed together,
too many different structs/interfaces depends each other.
* stability: due to the complexity & maintainability, sometimes there
are strange bugs and difficult to debug, and some code doesn't have test
(indeed some code is difficult to test because a lot of things are mixed
together).
* general applicability: although it is called "queue", its behavior is
not a well-known queue.
* scalability: it doesn't seem easy to make it work with a cluster
without breaking its behaviors.
It came from some very old code to "avoid breaking", however, its
technical debt is too heavy now. It's a good time to introduce a better
"queue" package.
# The new queue package
It keeps using old config and concept as much as possible.
* It only contains two major kinds of concepts:
* The "base queue": channel, levelqueue, redis
* They have the same abstraction, the same interface, and they are
tested by the same testing code.
* The "WokerPoolQueue", it uses the "base queue" to provide "worker
pool" function, calls the "handler" to process the data in the base
queue.
* The new code doesn't do "PushBack"
* Think about a queue with many workers, the "PushBack" can't guarantee
the order for re-queued unhandled items, so in new code it just does
"normal push"
* The new code doesn't do "pause/resume"
* The "pause/resume" was designed to handle some handler's failure: eg:
document indexer (elasticsearch) is down
* If a queue is paused for long time, either the producers blocks or the
new items are dropped.
* The new code doesn't do such "pause/resume" trick, it's not a common
queue's behavior and it doesn't help much.
* If there are unhandled items, the "push" function just blocks for a
few seconds and then re-queue them and retry.
* The new code doesn't do "worker booster"
* Gitea's queue's handlers are light functions, the cost is only the
go-routine, so it doesn't make sense to "boost" them.
* The new code only use "max worker number" to limit the concurrent
workers.
* The new "Push" never blocks forever
* Instead of creating more and more blocking goroutines, return an error
is more friendly to the server and to the end user.
There are more details in code comments: eg: the "Flush" problem, the
strange "code.index" hanging problem, the "immediate" queue problem.
Almost ready for review.
TODO:
* [x] add some necessary comments during review
* [x] add some more tests if necessary
* [x] update documents and config options
* [x] test max worker / active worker
* [x] re-run the CI tasks to see whether any test is flaky
* [x] improve the `handleOldLengthConfiguration` to provide more
friendly messages
* [x] fine tune default config values (eg: length?)
## Code coverage:
2023-05-08 07:49:59 -04:00
|
|
|
func PrintCurrentTest(t testing.TB, skip ...int) func() {
|
|
|
|
|
if len(skip) == 1 {
|
|
|
|
|
skip = []int{skip[0] + 1}
|
2022-09-02 15:18:23 -04:00
|
|
|
}
|
Rewrite queue (#24505)
# ⚠️ Breaking
Many deprecated queue config options are removed (actually, they should
have been removed in 1.18/1.19).
If you see the fatal message when starting Gitea: "Please update your
app.ini to remove deprecated config options", please follow the error
messages to remove these options from your app.ini.
Example:
```
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options
```
Many options in `[queue]` are are dropped, including:
`WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`,
`BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed
from app.ini.
# The problem
The old queue package has some legacy problems:
* complexity: I doubt few people could tell how it works.
* maintainability: Too many channels and mutex/cond are mixed together,
too many different structs/interfaces depends each other.
* stability: due to the complexity & maintainability, sometimes there
are strange bugs and difficult to debug, and some code doesn't have test
(indeed some code is difficult to test because a lot of things are mixed
together).
* general applicability: although it is called "queue", its behavior is
not a well-known queue.
* scalability: it doesn't seem easy to make it work with a cluster
without breaking its behaviors.
It came from some very old code to "avoid breaking", however, its
technical debt is too heavy now. It's a good time to introduce a better
"queue" package.
# The new queue package
It keeps using old config and concept as much as possible.
* It only contains two major kinds of concepts:
* The "base queue": channel, levelqueue, redis
* They have the same abstraction, the same interface, and they are
tested by the same testing code.
* The "WokerPoolQueue", it uses the "base queue" to provide "worker
pool" function, calls the "handler" to process the data in the base
queue.
* The new code doesn't do "PushBack"
* Think about a queue with many workers, the "PushBack" can't guarantee
the order for re-queued unhandled items, so in new code it just does
"normal push"
* The new code doesn't do "pause/resume"
* The "pause/resume" was designed to handle some handler's failure: eg:
document indexer (elasticsearch) is down
* If a queue is paused for long time, either the producers blocks or the
new items are dropped.
* The new code doesn't do such "pause/resume" trick, it's not a common
queue's behavior and it doesn't help much.
* If there are unhandled items, the "push" function just blocks for a
few seconds and then re-queue them and retry.
* The new code doesn't do "worker booster"
* Gitea's queue's handlers are light functions, the cost is only the
go-routine, so it doesn't make sense to "boost" them.
* The new code only use "max worker number" to limit the concurrent
workers.
* The new "Push" never blocks forever
* Instead of creating more and more blocking goroutines, return an error
is more friendly to the server and to the end user.
There are more details in code comments: eg: the "Flush" problem, the
strange "code.index" hanging problem, the "immediate" queue problem.
Almost ready for review.
TODO:
* [x] add some necessary comments during review
* [x] add some more tests if necessary
* [x] update documents and config options
* [x] test max worker / active worker
* [x] re-run the CI tasks to see whether any test is flaky
* [x] improve the `handleOldLengthConfiguration` to provide more
friendly messages
* [x] fine tune default config values (eg: length?)
## Code coverage:
2023-05-08 07:49:59 -04:00
|
|
|
return testlogger.PrintCurrentTest(t, skip...)
|
|
|
|
|
}
|
2022-12-22 18:41:56 -05:00
|
|
|
|
Rewrite queue (#24505)
# ⚠️ Breaking
Many deprecated queue config options are removed (actually, they should
have been removed in 1.18/1.19).
If you see the fatal message when starting Gitea: "Please update your
app.ini to remove deprecated config options", please follow the error
messages to remove these options from your app.ini.
Example:
```
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options
```
Many options in `[queue]` are are dropped, including:
`WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`,
`BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed
from app.ini.
# The problem
The old queue package has some legacy problems:
* complexity: I doubt few people could tell how it works.
* maintainability: Too many channels and mutex/cond are mixed together,
too many different structs/interfaces depends each other.
* stability: due to the complexity & maintainability, sometimes there
are strange bugs and difficult to debug, and some code doesn't have test
(indeed some code is difficult to test because a lot of things are mixed
together).
* general applicability: although it is called "queue", its behavior is
not a well-known queue.
* scalability: it doesn't seem easy to make it work with a cluster
without breaking its behaviors.
It came from some very old code to "avoid breaking", however, its
technical debt is too heavy now. It's a good time to introduce a better
"queue" package.
# The new queue package
It keeps using old config and concept as much as possible.
* It only contains two major kinds of concepts:
* The "base queue": channel, levelqueue, redis
* They have the same abstraction, the same interface, and they are
tested by the same testing code.
* The "WokerPoolQueue", it uses the "base queue" to provide "worker
pool" function, calls the "handler" to process the data in the base
queue.
* The new code doesn't do "PushBack"
* Think about a queue with many workers, the "PushBack" can't guarantee
the order for re-queued unhandled items, so in new code it just does
"normal push"
* The new code doesn't do "pause/resume"
* The "pause/resume" was designed to handle some handler's failure: eg:
document indexer (elasticsearch) is down
* If a queue is paused for long time, either the producers blocks or the
new items are dropped.
* The new code doesn't do such "pause/resume" trick, it's not a common
queue's behavior and it doesn't help much.
* If there are unhandled items, the "push" function just blocks for a
few seconds and then re-queue them and retry.
* The new code doesn't do "worker booster"
* Gitea's queue's handlers are light functions, the cost is only the
go-routine, so it doesn't make sense to "boost" them.
* The new code only use "max worker number" to limit the concurrent
workers.
* The new "Push" never blocks forever
* Instead of creating more and more blocking goroutines, return an error
is more friendly to the server and to the end user.
There are more details in code comments: eg: the "Flush" problem, the
strange "code.index" hanging problem, the "immediate" queue problem.
Almost ready for review.
TODO:
* [x] add some necessary comments during review
* [x] add some more tests if necessary
* [x] update documents and config options
* [x] test max worker / active worker
* [x] re-run the CI tasks to see whether any test is flaky
* [x] improve the `handleOldLengthConfiguration` to provide more
friendly messages
* [x] fine tune default config values (eg: length?)
## Code coverage:
2023-05-08 07:49:59 -04:00
|
|
|
// Printf takes a format and args and prints the string to os.Stdout
|
|
|
|
|
func Printf(format string, args ...interface{}) {
|
|
|
|
|
testlogger.Printf(format, args...)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
|
log.Register("test", testlogger.NewTestLogger)
|
2022-09-02 15:18:23 -04:00
|
|
|
}
|