foster/forgejo
1
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2025-01-04 14:48:59 -05:00
Code Issues Wiki Activity

Remove dependency on queue from setting

Browse source
This commit is contained in:
Andrew Thornton 2019-12-31 18:18:03 +00:00
parent 8798a61ba4
commit 030b6d91c8
No known key found for this signature in database
GPG key ID: 3CDE74631F13A748
7 changed files with 91 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
modules/indexer/issues/indexer.go
View file

@ -127,7 +127,7 @@ func InitIssueIndexer(syncReindex bool) {
}
}
issueIndexerQueue = setting.CreateQueue("issue_indexer", handler, &IndexerData{})
issueIndexerQueue = queue.CreateQueue("issue_indexer", handler, &IndexerData{})
if issueIndexerQueue == nil {
log.Fatal("Unable to create issue indexer queue")

4
modules/queue/queue.go
View file

@ -123,8 +123,8 @@ func RegisteredTypesAsString() []string {
return types
}
// CreateQueue takes a queue Type and HandlerFunc some options and possibly an exemplar and returns a Queue or an error
func CreateQueue(queueType Type, handlerFunc HandlerFunc, opts, exemplar interface{}) (Queue, error) {
// NewQueue takes a queue Type and HandlerFunc some options and possibly an exemplar and returns a Queue or an error
func NewQueue(queueType Type, handlerFunc HandlerFunc, opts, exemplar interface{}) (Queue, error) {
newFn, ok := queuesMap[queueType]
if !ok {
return nil, fmt.Errorf("Unsupported queue type: %v", queueType)

4
modules/queue/queue_wrapped.go
View file

@ -59,7 +59,7 @@ func (q *delayedStarter) setInternal(atShutdown func(context.Context, func()), h
q.lock.Unlock()
log.Fatal("Timedout creating queue %v with cfg %v in %s", q.underlying, q.cfg, q.name)
default:
queue, err := CreateQueue(q.underlying, handle, q.cfg, exemplar)
queue, err := NewQueue(q.underlying, handle, q.cfg, exemplar)
if err == nil {
q.internal = queue
q.lock.Unlock()
@ -101,7 +101,7 @@ func NewWrappedQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, erro
}
config := configInterface.(WrappedQueueConfiguration)
queue, err := CreateQueue(config.Underlying, handle, config.Config, exemplar)
queue, err := NewQueue(config.Underlying, handle, config.Config, exemplar)
if err == nil {
// Just return the queue there is no need to wrap
return queue, nil

75
modules/queue/setting.go Normal file
View file

@ -0,0 +1,75 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package queue
import (
"encoding/json"
"fmt"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)
func validType(t string) (Type, error) {
if len(t) == 0 {
return PersistableChannelQueueType, nil
}
for _, typ := range RegisteredTypes() {
if t == string(typ) {
return typ, nil
}
}
return PersistableChannelQueueType, fmt.Errorf("Unknown queue type: %s defaulting to %s", t, string(PersistableChannelQueueType))
}
// CreateQueue for name with provided handler and exemplar
func CreateQueue(name string, handle HandlerFunc, exemplar interface{}) Queue {
q := setting.GetQueueSettings(name)
opts := make(map[string]interface{})
opts["Name"] = name
opts["QueueLength"] = q.Length
opts["BatchLength"] = q.BatchLength
opts["DataDir"] = q.DataDir
opts["Addresses"] = q.Addresses
opts["Network"] = q.Network
opts["Password"] = q.Password
opts["DBIndex"] = q.DBIndex
opts["QueueName"] = q.QueueName
opts["Workers"] = q.Workers
opts["MaxWorkers"] = q.MaxWorkers
opts["BlockTimeout"] = q.BlockTimeout
opts["BoostTimeout"] = q.BoostTimeout
opts["BoostWorkers"] = q.BoostWorkers
typ, err := validType(q.Type)
if err != nil {
log.Error("Invalid type %s provided for queue named %s defaulting to %s", q.Type, name, string(typ))
}
cfg, err := json.Marshal(opts)
if err != nil {
log.Error("Unable to marshall generic options: %v Error: %v", opts, err)
log.Error("Unable to create queue for %s", name, err)
return nil
}
returnable, err := NewQueue(typ, handle, cfg, exemplar)
if q.WrapIfNecessary && err != nil {
log.Warn("Unable to create queue for %s: %v", name, err)
log.Warn("Attempting to create wrapped queue")
returnable, err = NewQueue(WrappedQueueType, handle, WrappedQueueConfiguration{
Underlying: Type(q.Type),
Timeout: q.Timeout,
MaxAttempts: q.MaxAttempts,
Config: cfg,
QueueLength: q.Length,
}, exemplar)
}
if err != nil {
log.Error("Unable to create queue for %s: %v", name, err)
return nil
}
return returnable
}

63
modules/setting/queue.go
View file

@ -5,7 +5,6 @@
package setting
import (
"encoding/json"
"fmt"
"path"
"strconv"
@ -13,10 +12,10 @@ import (
"time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/queue"
)
type queueSettings struct {
// QueueSettings represent the settings for a queue from the ini
type QueueSettings struct {
DataDir string
Length int
BatchLength int
@ -38,55 +37,11 @@ type queueSettings struct {
}
// Queue settings
var Queue = queueSettings{}
var Queue = QueueSettings{}
// CreateQueue for name with provided handler and exemplar
func CreateQueue(name string, handle queue.HandlerFunc, exemplar interface{}) queue.Queue {
q := getQueueSettings(name)
opts := make(map[string]interface{})
opts["Name"] = name
opts["QueueLength"] = q.Length
opts["BatchLength"] = q.BatchLength
opts["DataDir"] = q.DataDir
opts["Addresses"] = q.Addresses
opts["Network"] = q.Network
opts["Password"] = q.Password
opts["DBIndex"] = q.DBIndex
opts["QueueName"] = q.QueueName
opts["Workers"] = q.Workers
opts["MaxWorkers"] = q.MaxWorkers
opts["BlockTimeout"] = q.BlockTimeout
opts["BoostTimeout"] = q.BoostTimeout
opts["BoostWorkers"] = q.BoostWorkers
cfg, err := json.Marshal(opts)
if err != nil {
log.Error("Unable to marshall generic options: %v Error: %v", opts, err)
log.Error("Unable to create queue for %s", name, err)
return nil
}
returnable, err := queue.CreateQueue(queue.Type(q.Type), handle, cfg, exemplar)
if q.WrapIfNecessary && err != nil {
log.Warn("Unable to create queue for %s: %v", name, err)
log.Warn("Attempting to create wrapped queue")
returnable, err = queue.CreateQueue(queue.WrappedQueueType, handle, queue.WrappedQueueConfiguration{
Underlying: queue.Type(q.Type),
Timeout: q.Timeout,
MaxAttempts: q.MaxAttempts,
Config: cfg,
QueueLength: q.Length,
}, exemplar)
}
if err != nil {
log.Error("Unable to create queue for %s: %v", name, err)
return nil
}
return returnable
}
func getQueueSettings(name string) queueSettings {
q := queueSettings{}
// GetQueueSettings returns the queue settings for the appropriately named queue
func GetQueueSettings(name string) QueueSettings {
q := QueueSettings{}
sec := Cfg.Section("queue." + name)
// DataDir is not directly inheritable
q.DataDir = path.Join(Queue.DataDir, name)
@ -104,8 +59,7 @@ func getQueueSettings(name string) queueSettings {
q.Length = sec.Key("LENGTH").MustInt(Queue.Length)
q.BatchLength = sec.Key("BATCH_LENGTH").MustInt(Queue.BatchLength)
q.ConnectionString = sec.Key("CONN_STR").MustString(Queue.ConnectionString)
validTypes := queue.RegisteredTypesAsString()
q.Type = sec.Key("TYPE").In(Queue.Type, validTypes)
q.Type = sec.Key("TYPE").MustString(Queue.Type)
q.WrapIfNecessary = sec.Key("WRAP_IF_NECESSARY").MustBool(Queue.WrapIfNecessary)
q.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Queue.MaxAttempts)
q.Timeout = sec.Key("TIMEOUT").MustDuration(Queue.Timeout)
@ -131,8 +85,7 @@ func NewQueueService() {
Queue.Length = sec.Key("LENGTH").MustInt(20)
Queue.BatchLength = sec.Key("BATCH_LENGTH").MustInt(20)
Queue.ConnectionString = sec.Key("CONN_STR").MustString(path.Join(AppDataPath, ""))
validTypes := queue.RegisteredTypesAsString()
Queue.Type = sec.Key("TYPE").In(string(queue.PersistableChannelQueueType), validTypes)
Queue.Type = sec.Key("TYPE").MustString("")
Queue.Network, Queue.Addresses, Queue.Password, Queue.DBIndex, _ = ParseQueueConnStr(Queue.ConnectionString)
Queue.WrapIfNecessary = sec.Key("WRAP_IF_NECESSARY").MustBool(true)
Queue.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(10)

6
modules/setting/task.go
View file

@ -4,16 +4,14 @@
package setting
import "code.gitea.io/gitea/modules/queue"
func newTaskService() {
taskSec := Cfg.Section("task")
queueTaskSec := Cfg.Section("queue.task")
switch taskSec.Key("QUEUE_TYPE").MustString(ChannelQueueType) {
case ChannelQueueType:
queueTaskSec.Key("TYPE").MustString(string(queue.PersistableChannelQueueType))
queueTaskSec.Key("TYPE").MustString("persistable-channel")
case RedisQueueType:
queueTaskSec.Key("TYPE").MustString(string(queue.RedisQueueType))
queueTaskSec.Key("TYPE").MustString("redis")
}
queueTaskSec.Key("LENGTH").MustInt(taskSec.Key("QUEUE_LENGTH").MustInt(1000))
queueTaskSec.Key("CONN_STR").MustString(taskSec.Key("QUEUE_CONN_STR").MustString("addrs=127.0.0.1:6379 db=0"))

3
modules/task/task.go
View file

@ -12,7 +12,6 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/queue"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
)
@ -31,7 +30,7 @@ func Run(t *models.Task) error {
// Init will start the service to get all unfinished tasks and run them
func Init() error {
taskQueue = setting.CreateQueue("task", handle, &models.Task{})
taskQueue = queue.CreateQueue("task", handle, &models.Task{})
if taskQueue == nil {
return fmt.Errorf("Unable to create Task Queue")