foster/forgejo
1
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2024-12-21 12:44:49 -05:00
Code Issues Wiki Activity

LFS support to be stored on minio (#12518)

Browse source 
* LFS support to be stored on minio

* Fix test

* Fix lint

* Fix lint

* Fix check

* Fix test

* Update documents and add migration for LFS

* Fix some bugs
This commit is contained in:
Lunny Xiao 2020-09-08 23:45:10 +08:00 committed by GitHub
parent e4b3f35b8d
commit 7a5465fc56
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 423 additions and 203 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

76
cmd/migrate_storage.go
View file

@ -83,6 +83,13 @@ func migrateAttachments(dstStorage storage.ObjectStorage) error {
})
}
func migrateLFS(dstStorage storage.ObjectStorage) error {
return models.IterateLFS(func(mo *models.LFSMetaObject) error {
_, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath())
return err
})
}
func runMigrateStorage(ctx *cli.Context) error {
if err := initDB(); err != nil {
return err
@ -103,45 +110,50 @@ func runMigrateStorage(ctx *cli.Context) error {
return err
}
var dstStorage storage.ObjectStorage
var err error
switch ctx.String("store") {
case "local":
p := ctx.String("path")
if p == "" {
log.Fatal("Path must be given when store is loal")
return nil
}
dstStorage, err = storage.NewLocalStorage(p)
case "minio":
dstStorage, err = storage.NewMinioStorage(
context.Background(),
ctx.String("minio-endpoint"),
ctx.String("minio-access-key-id"),
ctx.String("minio-secret-access-key"),
ctx.String("minio-bucket"),
ctx.String("minio-location"),
ctx.String("minio-base-path"),
ctx.Bool("minio-use-ssl"),
)
default:
return fmt.Errorf("Unsupported attachments store type: %s", ctx.String("store"))
}
if err != nil {
return err
}
tp := ctx.String("type")
switch tp {
case "attachments":
var dstStorage storage.ObjectStorage
var err error
switch ctx.String("store") {
case "local":
p := ctx.String("path")
if p == "" {
log.Fatal("Path must be given when store is loal")
return nil
}
dstStorage, err = storage.NewLocalStorage(p)
case "minio":
dstStorage, err = storage.NewMinioStorage(
context.Background(),
ctx.String("minio-endpoint"),
ctx.String("minio-access-key-id"),
ctx.String("minio-secret-access-key"),
ctx.String("minio-bucket"),
ctx.String("minio-location"),
ctx.String("minio-base-path"),
ctx.Bool("minio-use-ssl"),
)
default:
return fmt.Errorf("Unsupported attachments store type: %s", ctx.String("store"))
}
if err != nil {
return err
}
if err := migrateAttachments(dstStorage); err != nil {
return err
}
log.Warn("All files have been copied to the new placement but old files are still on the orignial placement.")
return nil
case "lfs":
if err := migrateLFS(dstStorage); err != nil {
return err
}
default:
return fmt.Errorf("Unsupported storage: %s", ctx.String("type"))
}
log.Warn("All files have been copied to the new placement but old files are still on the orignial placement.")
return nil
}

13
docs/content/doc/advanced/config-cheat-sheet.en-us.md
View file

@ -206,12 +206,23 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
- `STATIC_CACHE_TIME`: **6h**: Web browser cache time for static resources on `custom/`, `public/` and all uploaded avatars.
- `ENABLE_GZIP`: **false**: Enables application-level GZIP support.
- `LANDING_PAGE`: **home**: Landing page for unauthenticated users \[home, explore, organizations, login\].
- `LFS_START_SERVER`: **false**: Enables git-lfs support.
- `LFS_CONTENT_PATH`: **./data/lfs**: Where to store LFS files.
- `LFS_STORE_TYPE`: **local**: Storage type for lfs, `local` for local disk or `minio` for s3 compatible object storage service.
- `LFS_SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
- `LFS_CONTENT_PATH`: **./data/lfs**: Where to store LFS files, only available when `LFS_STORE_TYPE` is `local`.
- `LFS_MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `LFS_STORE_TYPE` is `minio`
- `LFS_MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `LFS_STORE_TYPE` is `minio`
- `LFS_MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `LFS_STORE_TYPE is` `minio`
- `LFS_MINIO_BUCKET`: **gitea**: Minio bucket to store the lfs only available when `LFS_STORE_TYPE` is `minio`
- `LFS_MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `LFS_STORE_TYPE` is `minio`
- `LFS_MINIO_BASE_PATH`: **lfs/**: Minio base path on the bucket only available when `LFS_STORE_TYPE` is `minio`
- `LFS_MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `LFS_STORE_TYPE` is `minio`
- `LFS_JWT_SECRET`: **\<empty\>**: LFS authentication secret, change this a unique string.
- `LFS_HTTP_AUTH_EXPIRY`: **20m**: LFS authentication validity period in time.Duration, pushes taking longer than this may fail.
- `LFS_MAX_FILE_SIZE`: **0**: Maximum allowed LFS file size in bytes (Set to 0 for no limit).
- `LFS_LOCK_PAGING_NUM`: **50**: Maximum number of LFS Locks returned per page.
- `REDIRECT_OTHER_PORT`: **false**: If true and `PROTOCOL` is https, allows redirecting http requests on `PORT_TO_REDIRECT` to the https port Gitea listens on.
- `PORT_TO_REDIRECT`: **80**: Port for the http redirection service to listen on. Used when `REDIRECT_OTHER_PORT` is true.
- `ENABLE_LETSENCRYPT`: **false**: If enabled you must set `DOMAIN` to valid internet facing domain (ensure DNS is set and port 80 is accessible by letsencrypt validation server).

10
docs/content/doc/advanced/config-cheat-sheet.zh-cn.md
View file

@ -69,8 +69,18 @@ menu:
- `STATIC_CACHE_TIME`: **6h**: 静态资源文件，包括 `custom/`, `public/` 和所有上传的头像的浏览器缓存时间。
- `ENABLE_GZIP`: 启用应用级别的 GZIP 压缩。
- `LANDING_PAGE`: 未登录用户的默认页面，可选 `home` 或 `explore`。
- `LFS_START_SERVER`: 是否启用 git-lfs 支持. 可以为 `true` 或 `false`， 默认是 `false`。
- `LFS_STORE_TYPE`: **local**: LFS 的存储类型，`local` 将存储到磁盘，`minio` 将存储到 s3 兼容的对象服务。
- `LFS_SERVE_DIRECT`: **false**: 允许直接重定向到存储系统。当前，仅 Minio/S3 是支持的。
- `LFS_CONTENT_PATH`: 存放 lfs 命令上传的文件的地方，默认是 `data/lfs`。
- `LFS_MINIO_ENDPOINT`: **localhost:9000**: Minio 地址，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
- `LFS_MINIO_ACCESS_KEY_ID`: Minio accessKeyID，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
- `LFS_MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
- `LFS_MINIO_BUCKET`: **gitea**: Minio bucket，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
- `LFS_MINIO_LOCATION`: **us-east-1**: Minio location ，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
- `LFS_MINIO_BASE_PATH`: **lfs/**: Minio base path ，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
- `LFS_MINIO_USE_SSL`: **false**: Minio 是否启用 ssl ，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
- `LFS_JWT_SECRET`: LFS 认证密钥，改成自己的。
## Database (`database`)

7
integrations/lfs_getobject_test.go
View file

@ -18,6 +18,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"gitea.com/macaron/gzip"
gzipp "github.com/klauspost/compress/gzip"
@ -49,8 +50,10 @@ func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string
lfsID++
lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject)
assert.NoError(t, err)
contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath}
if !contentStore.Exists(lfsMetaObject) {
contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS}
exist, err := contentStore.Exists(lfsMetaObject)
assert.NoError(t, err)
if !exist {
err := contentStore.Put(lfsMetaObject, bytes.NewReader(*content))
assert.NoError(t, err)
}

20
integrations/mysql.ini.tmpl
View file

@ -36,13 +36,23 @@ ROOT_URL = http://localhost:3001/
DISABLE_SSH = false
SSH_LISTEN_HOST = localhost
SSH_PORT = 2201
START_SSH_SERVER = true
LFS_START_SERVER = true
LFS_CONTENT_PATH = integrations/gitea-integration-mysql/datalfs-mysql
OFFLINE_MODE = false
LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
APP_DATA_PATH = integrations/gitea-integration-mysql/data
BUILTIN_SSH_SERVER_USER = git
START_SSH_SERVER = true
OFFLINE_MODE = false
LFS_START_SERVER = true
LFS_CONTENT_PATH = integrations/gitea-integration-mysql/datalfs-mysql
LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
LFS_STORE_TYPE = minio
LFS_SERVE_DIRECT = false
LFS_MINIO_ENDPOINT = minio:9000
LFS_MINIO_ACCESS_KEY_ID = 123456
LFS_MINIO_SECRET_ACCESS_KEY = 12345678
LFS_MINIO_BUCKET = gitea
LFS_MINIO_LOCATION = us-east-1
LFS_MINIO_BASE_PATH = lfs/
LFS_MINIO_USE_SSL = false
[attachment]
STORE_TYPE = minio

32
models/lfs.go
View file

@ -10,6 +10,7 @@ import (
"errors"
"fmt"
"io"
"path"
"code.gitea.io/gitea/modules/timeutil"
@ -26,6 +27,15 @@ type LFSMetaObject struct {
CreatedUnix timeutil.TimeStamp `xorm:"created"`
}
// RelativePath returns the relative path of the lfs object
func (m *LFSMetaObject) RelativePath() string {
if len(m.Oid) < 5 {
return m.Oid
}
return path.Join(m.Oid[0:2], m.Oid[2:4], m.Oid[4:])
}
// Pointer returns the string representation of an LFS pointer file
func (m *LFSMetaObject) Pointer() string {
return fmt.Sprintf("%s\n%s%s\nsize %d\n", LFSMetaFileIdentifier, LFSMetaFileOidPrefix, m.Oid, m.Size)
@ -202,3 +212,25 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *User, repoID int64) error {
return sess.Commit()
}
// IterateLFS iterates lfs object
func IterateLFS(f func(mo *LFSMetaObject) error) error {
var start int
const batchSize = 100
for {
var mos = make([]*LFSMetaObject, 0, batchSize)
if err := x.Limit(batchSize, start).Find(&mos); err != nil {
return err
}
if len(mos) == 0 {
return nil
}
start += len(mos)
for _, mo := range mos {
if err := f(mo); err != nil {
return err
}
}
}
}

1
models/unit_tests.go
View file

@ -69,6 +69,7 @@ func MainTest(m *testing.M, pathToGiteaRoot string) {
}
setting.Attachment.Path = filepath.Join(setting.AppDataPath, "attachments")
setting.LFS.ContentPath = filepath.Join(setting.AppDataPath, "lfs")
if err = storage.Init(); err != nil {
fatalTestError("storage.Init: %v\n", err)
}

82
modules/lfs/content_store.go
View file

@ -10,11 +10,10 @@ import (
"errors"
"io"
"os"
"path/filepath"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/storage"
)
var (
@ -24,17 +23,15 @@ var (
// ContentStore provides a simple file system based storage.
type ContentStore struct {
BasePath string
storage.ObjectStorage
}
// Get takes a Meta object and retrieves the content from the store, returning
// it as an io.Reader. If fromByte > 0, the reader starts from that byte
func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) {
path := filepath.Join(s.BasePath, transformKey(meta.Oid))
f, err := os.Open(path)
f, err := s.Open(meta.RelativePath())
if err != nil {
log.Error("Whilst trying to read LFS OID[%s]: Unable to open %s Error: %v", meta.Oid, path, err)
log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", meta.Oid, err)
return nil, err
}
if fromByte > 0 {
@ -48,82 +45,55 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC
// Put takes a Meta object and an io.Reader and writes the content to the store.
func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error {
path := filepath.Join(s.BasePath, transformKey(meta.Oid))
tmpPath := path + ".tmp"
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0750); err != nil {
log.Error("Whilst putting LFS OID[%s]: Unable to create the LFS directory: %s Error: %v", meta.Oid, dir, err)
return err
}
file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0640)
if err != nil {
log.Error("Whilst putting LFS OID[%s]: Unable to open temporary file for writing: %s Error: %v", tmpPath, err)
return err
}
defer func() {
if err := util.Remove(tmpPath); err != nil {
log.Warn("Unable to remove temporary path: %s: Error: %v", tmpPath, err)
}
}()
hash := sha256.New()
hw := io.MultiWriter(hash, file)
written, err := io.Copy(hw, r)
rd := io.TeeReader(r, hash)
p := meta.RelativePath()
written, err := s.Save(p, rd)
if err != nil {
log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, tmpPath, err)
file.Close()
log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, p, err)
return err
}
file.Close()
if written != meta.Size {
if err := s.Delete(p); err != nil {
log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
}
return errSizeMismatch
}
shaStr := hex.EncodeToString(hash.Sum(nil))
if shaStr != meta.Oid {
if err := s.Delete(p); err != nil {
log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
}
return errHashMismatch
}
if err := os.Rename(tmpPath, path); err != nil {
log.Error("Whilst putting LFS OID[%s]: Unable to move tmp file to final destination: %s Error: %v", meta.Oid, path, err)
return err
}
return nil
}
// Exists returns true if the object exists in the content store.
func (s *ContentStore) Exists(meta *models.LFSMetaObject) bool {
path := filepath.Join(s.BasePath, transformKey(meta.Oid))
if _, err := os.Stat(path); os.IsNotExist(err) {
return false
func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) {
_, err := s.ObjectStorage.Stat(meta.RelativePath())
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
return true
return true, nil
}
// Verify returns true if the object exists in the content store and size is correct.
func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) {
path := filepath.Join(s.BasePath, transformKey(meta.Oid))
fi, err := os.Stat(path)
if os.IsNotExist(err) || err == nil && fi.Size() != meta.Size {
p := meta.RelativePath()
fi, err := s.ObjectStorage.Stat(p)
if os.IsNotExist(err) || (err == nil && fi.Size() != meta.Size) {
return false, nil
} else if err != nil {
log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", path, meta.Oid, err)
log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", p, meta.Oid, err)
return false, err
}
return true, nil
}
func transformKey(key string) string {
if len(key) < 5 {
return key
}
return filepath.Join(key[0:2], key[2:4], key[4:])
}

8
modules/lfs/pointers.go
View file

@ -12,6 +12,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
)
// ReadPointerFile will return a partially filled LFSMetaObject if the provided reader is a pointer file
@ -53,9 +54,10 @@ func IsPointerFile(buf *[]byte) *models.LFSMetaObject {
return nil
}
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
contentStore := &ContentStore{ObjectStorage: storage.LFS}
meta := &models.LFSMetaObject{Oid: oid, Size: size}
if !contentStore.Exists(meta) {
exist, err := contentStore.Exists(meta)
if err != nil || !exist {
return nil
}
@ -64,6 +66,6 @@ func IsPointerFile(buf *[]byte) *models.LFSMetaObject {
// ReadMetaObject will read a models.LFSMetaObject and return a reader
func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) {
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
contentStore := &ContentStore{ObjectStorage: storage.LFS}
return contentStore.Get(meta, 0)
}

41
modules/lfs/server.go
View file

@ -20,6 +20,7 @@ import (
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"gitea.com/macaron/macaron"
"github.com/dgrijalva/jwt-go"
@ -187,7 +188,7 @@ func getContentHandler(ctx *context.Context) {
}
}
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
contentStore := &ContentStore{ObjectStorage: storage.LFS}
content, err := contentStore.Get(meta, fromByte)
if err != nil {
// Errors are logged in contentStore.Get
@ -288,8 +289,14 @@ func PostHandler(ctx *context.Context) {
ctx.Resp.Header().Set("Content-Type", metaMediaType)
sentStatus := 202
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
if meta.Existing && contentStore.Exists(meta) {
contentStore := &ContentStore{ObjectStorage: storage.LFS}
exist, err := contentStore.Exists(meta)
if err != nil {
log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", rv.Oid, rv.User, rv.Repo, err)
writeStatus(ctx, 500)
return
}
if meta.Existing && exist {
sentStatus = 200
}
ctx.Resp.WriteHeader(sentStatus)
@ -343,12 +350,20 @@ func BatchHandler(ctx *context.Context) {
return
}
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
contentStore := &ContentStore{ObjectStorage: storage.LFS}
meta, err := repository.GetLFSMetaObjectByOid(object.Oid)
if err == nil && contentStore.Exists(meta) { // Object is found and exists
responseObjects = append(responseObjects, Represent(object, meta, true, false))
continue
if err == nil { // Object is found and exists
exist, err := contentStore.Exists(meta)
if err != nil {
log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err)
writeStatus(ctx, 500)
return
}
if exist {
responseObjects = append(responseObjects, Represent(object, meta, true, false))
continue
}
}
if requireWrite && setting.LFS.MaxFileSize > 0 && object.Size > setting.LFS.MaxFileSize {
@ -360,7 +375,13 @@ func BatchHandler(ctx *context.Context) {
// Object is not found
meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID})
if err == nil {
responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, !contentStore.Exists(meta)))
exist, err := contentStore.Exists(meta)
if err != nil {
log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err)
writeStatus(ctx, 500)
return
}
responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, !exist))
} else {
log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", object.Oid, object.Size, object.User, object.Repo, err)
}
@ -387,7 +408,7 @@ func PutHandler(ctx *context.Context) {
return
}
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
contentStore := &ContentStore{ObjectStorage: storage.LFS}
bodyReader := ctx.Req.Body().ReadCloser()
defer bodyReader.Close()
if err := contentStore.Put(meta, bodyReader); err != nil {
@ -429,7 +450,7 @@ func VerifyHandler(ctx *context.Context) {
return
}
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
contentStore := &ContentStore{ObjectStorage: storage.LFS}
ok, err := contentStore.Verify(meta)
if err != nil {
// Error will be logged in Verify

9
modules/repofiles/update.go
View file

@ -20,6 +20,7 @@ import (
"code.gitea.io/gitea/modules/log"
repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/structs"
pull_service "code.gitea.io/gitea/services/pull"
@ -433,8 +434,12 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up
if err != nil {
return nil, err
}
contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath}
if !contentStore.Exists(lfsMetaObject) {
contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS}
exist, err := contentStore.Exists(lfsMetaObject)
if err != nil {
return nil, err
}
if !exist {
if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil {
if _, err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil {
return nil, fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err)

9
modules/repofiles/upload.go
View file

@ -13,6 +13,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
)
// UploadRepoFileOptions contains the uploaded repository file options
@ -163,12 +164,16 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRep
// OK now we can insert the data into the store - there's no way to clean up the store
// once it's in there, it's in there.
contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath}
contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS}
for _, uploadInfo := range infos {
if uploadInfo.lfsMetaObject == nil {
continue
}
if !contentStore.Exists(uploadInfo.lfsMetaObject) {
exist, err := contentStore.Exists(uploadInfo.lfsMetaObject)
if err != nil {
return cleanUpAfterFailure(&infos, t, err)
}
if !exist {
file, err := os.Open(uploadInfo.upload.LocalPath())
if err != nil {
return cleanUpAfterFailure(&infos, t, err)

122
modules/setting/lfs.go Normal file
View file

@ -0,0 +1,122 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package setting
import (
"encoding/base64"
"os"
"path/filepath"
"time"
"code.gitea.io/gitea/modules/generate"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"github.com/unknwon/com"
ini "gopkg.in/ini.v1"
)
// LFS represents the configuration for Git LFS
var LFS = struct {
StartServer bool `ini:"LFS_START_SERVER"`
ContentPath string `ini:"LFS_CONTENT_PATH"`
JWTSecretBase64 string `ini:"LFS_JWT_SECRET"`
JWTSecretBytes []byte `ini:"-"`
HTTPAuthExpiry time.Duration `ini:"LFS_HTTP_AUTH_EXPIRY"`
MaxFileSize int64 `ini:"LFS_MAX_FILE_SIZE"`
LocksPagingNum int `ini:"LFS_LOCKS_PAGING_NUM"`
StoreType string
ServeDirect bool
Minio struct {
Endpoint string
AccessKeyID string
SecretAccessKey string
UseSSL bool
Bucket string
Location string
BasePath string
}
}{
StoreType: "local",
}
func newLFSService() {
sec := Cfg.Section("server")
if err := sec.MapTo(&LFS); err != nil {
log.Fatal("Failed to map LFS settings: %v", err)
}
LFS.ContentPath = sec.Key("LFS_CONTENT_PATH").MustString(filepath.Join(AppDataPath, "lfs"))
if !filepath.IsAbs(LFS.ContentPath) {
LFS.ContentPath = filepath.Join(AppWorkPath, LFS.ContentPath)
}
if LFS.LocksPagingNum == 0 {
LFS.LocksPagingNum = 50
}
LFS.HTTPAuthExpiry = sec.Key("LFS_HTTP_AUTH_EXPIRY").MustDuration(20 * time.Minute)
if LFS.StartServer {
LFS.JWTSecretBytes = make([]byte, 32)
n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64))
if err != nil || n != 32 {
LFS.JWTSecretBase64, err = generate.NewJwtSecret()
if err != nil {
log.Fatal("Error generating JWT Secret for custom config: %v", err)
return
}
// Save secret
cfg := ini.Empty()
if com.IsFile(CustomConf) {
// Keeps custom settings if there is already something.
if err := cfg.Append(CustomConf); err != nil {
log.Error("Failed to load custom conf '%s': %v", CustomConf, err)
}
}
cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64)
if err := os.MkdirAll(filepath.Dir(CustomConf), os.ModePerm); err != nil {
log.Fatal("Failed to create '%s': %v", CustomConf, err)
}
if err := cfg.SaveTo(CustomConf); err != nil {
log.Fatal("Error saving generated JWT Secret to custom config: %v", err)
return
}
}
}
}
func ensureLFSDirectory() {
if LFS.StartServer {
if err := os.MkdirAll(LFS.ContentPath, 0700); err != nil {
log.Fatal("Failed to create '%s': %v", LFS.ContentPath, err)
}
}
}
// CheckLFSVersion will check lfs version, if not satisfied, then disable it.
func CheckLFSVersion() {
if LFS.StartServer {
//Disable LFS client hooks if installed for the current OS user
//Needs at least git v2.1.2
err := git.LoadGitVersion()
if err != nil {
log.Fatal("Error retrieving git version: %v", err)
}
if git.CheckGitVersionConstraint(">= 2.1.2") != nil {
LFS.StartServer = false
log.Error("LFS server support needs at least Git v2.1.2")
} else {
git.GlobalCommandArgs = append(git.GlobalCommandArgs, "-c", "filter.lfs.required=",
"-c", "filter.lfs.smudge=", "-c", "filter.lfs.clean=")
}
}
}

86
modules/setting/setting.go
View file

@ -23,7 +23,6 @@ import (
"time"
"code.gitea.io/gitea/modules/generate"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/user"
@ -133,16 +132,6 @@ var (
MinimumKeySizes: map[string]int{"ed25519": 256, "ecdsa": 256, "rsa": 2048, "dsa": 1024},
}
LFS struct {
StartServer bool `ini:"LFS_START_SERVER"`
ContentPath string `ini:"LFS_CONTENT_PATH"`
JWTSecretBase64 string `ini:"LFS_JWT_SECRET"`
JWTSecretBytes []byte `ini:"-"`
HTTPAuthExpiry time.Duration `ini:"LFS_HTTP_AUTH_EXPIRY"`
MaxFileSize int64 `ini:"LFS_MAX_FILE_SIZE"`
LocksPagingNum int `ini:"LFS_LOCKS_PAGING_NUM"`
}
// Security settings
InstallLock bool
SecretKey string
@ -472,27 +461,6 @@ func createPIDFile(pidPath string) {
}
}
// CheckLFSVersion will check lfs version, if not satisfied, then disable it.
func CheckLFSVersion() {
if LFS.StartServer {
//Disable LFS client hooks if installed for the current OS user
//Needs at least git v2.1.2
err := git.LoadGitVersion()
if err != nil {
log.Fatal("Error retrieving git version: %v", err)
}
if git.CheckGitVersionConstraint(">= 2.1.2") != nil {
LFS.StartServer = false
log.Error("LFS server support needs at least Git v2.1.2")
} else {
git.GlobalCommandArgs = append(git.GlobalCommandArgs, "-c", "filter.lfs.required=",
"-c", "filter.lfs.smudge=", "-c", "filter.lfs.clean=")
}
}
}
// SetCustomPathAndConf will set CustomPath and CustomConf with reference to the
// GITEA_CUSTOM environment variable and with provided overrides before stepping
// back to the default
@ -722,51 +690,7 @@ func NewContext() {
SSH.CreateAuthorizedKeysFile = sec.Key("SSH_CREATE_AUTHORIZED_KEYS_FILE").MustBool(true)
SSH.ExposeAnonymous = sec.Key("SSH_EXPOSE_ANONYMOUS").MustBool(false)
sec = Cfg.Section("server")
if err = sec.MapTo(&LFS); err != nil {
log.Fatal("Failed to map LFS settings: %v", err)
}
LFS.ContentPath = sec.Key("LFS_CONTENT_PATH").MustString(filepath.Join(AppDataPath, "lfs"))
if !filepath.IsAbs(LFS.ContentPath) {
LFS.ContentPath = filepath.Join(AppWorkPath, LFS.ContentPath)
}
if LFS.LocksPagingNum == 0 {
LFS.LocksPagingNum = 50
}
LFS.HTTPAuthExpiry = sec.Key("LFS_HTTP_AUTH_EXPIRY").MustDuration(20 * time.Minute)
if LFS.StartServer {
LFS.JWTSecretBytes = make([]byte, 32)
n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64))
if err != nil || n != 32 {
LFS.JWTSecretBase64, err = generate.NewJwtSecret()
if err != nil {
log.Fatal("Error generating JWT Secret for custom config: %v", err)
return
}
// Save secret
cfg := ini.Empty()
if com.IsFile(CustomConf) {
// Keeps custom settings if there is already something.
if err := cfg.Append(CustomConf); err != nil {
log.Error("Failed to load custom conf '%s': %v", CustomConf, err)
}
}
cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64)
if err := os.MkdirAll(filepath.Dir(CustomConf), os.ModePerm); err != nil {
log.Fatal("Failed to create '%s': %v", CustomConf, err)
}
if err := cfg.SaveTo(CustomConf); err != nil {
log.Fatal("Error saving generated JWT Secret to custom config: %v", err)
return
}
}
}
newLFSService()
if err = Cfg.Section("oauth2").MapTo(&OAuth2); err != nil {
log.Fatal("Failed to OAuth2 settings: %v", err)
@ -1086,14 +1010,6 @@ func loadOrGenerateInternalToken(sec *ini.Section) string {
return token
}
func ensureLFSDirectory() {
if LFS.StartServer {
if err := os.MkdirAll(LFS.ContentPath, 0700); err != nil {
log.Fatal("Failed to create '%s': %v", LFS.ContentPath, err)
}
}
}
// NewServices initializes the services
func NewServices() {
InitDBConfig()

7
modules/storage/local.go
View file

@ -34,7 +34,7 @@ func NewLocalStorage(bucket string) (*LocalStorage, error) {
}
// Open a file
func (l *LocalStorage) Open(path string) (io.ReadCloser, error) {
func (l *LocalStorage) Open(path string) (Object, error) {
return os.Open(filepath.Join(l.dir, path))
}
@ -58,6 +58,11 @@ func (l *LocalStorage) Save(path string, r io.Reader) (int64, error) {
return io.Copy(f, r)
}
// Stat returns the info of the file
func (l *LocalStorage) Stat(path string) (ObjectInfo, error) {
return os.Stat(filepath.Join(l.dir, path))
}
// Delete delete a file
func (l *LocalStorage) Delete(path string) error {
p := filepath.Join(l.dir, path)

38
modules/storage/minio.go
View file

@ -8,6 +8,7 @@ import (
"context"
"io"
"net/url"
"os"
"path"
"strings"
"time"
@ -62,7 +63,7 @@ func (m *MinioStorage) buildMinioPath(p string) string {
}
// Open open a file
func (m *MinioStorage) Open(path string) (io.ReadCloser, error) {
func (m *MinioStorage) Open(path string) (Object, error) {
var opts = minio.GetObjectOptions{}
object, err := m.client.GetObject(m.ctx, m.bucket, m.buildMinioPath(path), opts)
if err != nil {
@ -87,6 +88,41 @@ func (m *MinioStorage) Save(path string, r io.Reader) (int64, error) {
return uploadInfo.Size, nil
}
type minioFileInfo struct {
minio.ObjectInfo
}
func (m minioFileInfo) Name() string {
return m.ObjectInfo.Key
}
func (m minioFileInfo) Size() int64 {
return m.ObjectInfo.Size
}
func (m minioFileInfo) ModTime() time.Time {
return m.LastModified
}
// Stat returns the stat information of the object
func (m *MinioStorage) Stat(path string) (ObjectInfo, error) {
info, err := m.client.StatObject(
m.ctx,
m.bucket,
m.buildMinioPath(path),
minio.StatObjectOptions{},
)
if err != nil {
if errResp, ok := err.(minio.ErrorResponse); ok {
if errResp.Code == "NoSuchKey" {
return nil, os.ErrNotExist
}
}
return nil, err
}
return &minioFileInfo{info}, nil
}
// Delete delete a file
func (m *MinioStorage) Delete(path string) error {
return m.client.RemoveObject(m.ctx, m.bucket, m.buildMinioPath(path), minio.RemoveObjectOptions{})

56
modules/storage/storage.go
View file

@ -10,6 +10,7 @@ import (
"fmt"
"io"
"net/url"
"time"
"code.gitea.io/gitea/modules/setting"
)
@ -19,10 +20,24 @@ var (
ErrURLNotSupported = errors.New("url method not supported")
)
// Object represents the object on the storage
type Object interface {
io.ReadCloser
io.Seeker
}
// ObjectInfo represents the object info on the storage
type ObjectInfo interface {
Name() string
Size() int64
ModTime() time.Time
}
// ObjectStorage represents an object storage to handle a bucket and files
type ObjectStorage interface {
Open(path string) (Object, error)
Save(path string, r io.Reader) (int64, error)
Open(path string) (io.ReadCloser, error)
Stat(path string) (ObjectInfo, error)
Delete(path string) error
URL(path, name string) (*url.URL, error)
}
@ -41,10 +56,21 @@ func Copy(dstStorage ObjectStorage, dstPath string, srcStorage ObjectStorage, sr
var (
// Attachments represents attachments storage
Attachments ObjectStorage
// LFS represents lfs storage
LFS ObjectStorage
)
// Init init the stoarge
func Init() error {
if err := initAttachments(); err != nil {
return err
}
return initLFS()
}
func initAttachments() error {
var err error
switch setting.Attachment.StoreType {
case "local":
@ -71,3 +97,31 @@ func Init() error {
return nil
}
func initLFS() error {
var err error
switch setting.LFS.StoreType {
case "local":
LFS, err = NewLocalStorage(setting.LFS.ContentPath)
case "minio":
minio := setting.LFS.Minio
LFS, err = NewMinioStorage(
context.Background(),
minio.Endpoint,
minio.AccessKeyID,
minio.SecretAccessKey,
minio.Bucket,
minio.Location,
minio.BasePath,
minio.UseSSL,
)
default:
return fmt.Errorf("Unsupported LFS store type: %s", setting.LFS.StoreType)
}
if err != nil {
return err
}
return nil
}

9
routers/repo/lfs.go
View file

@ -28,6 +28,7 @@ import (
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/util"
gogit "github.com/go-git/go-git/v5"
@ -619,7 +620,7 @@ type pointerResult struct {
func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) {
defer wg.Done()
defer catFileBatchReader.Close()
contentStore := lfs.ContentStore{BasePath: setting.LFS.ContentPath}
contentStore := lfs.ContentStore{ObjectStorage: storage.LFS}
bufferedReader := bufio.NewReader(catFileBatchReader)
buf := make([]byte, 1025)
@ -673,7 +674,11 @@ func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg
result.InRepo = true
}
result.Exists = contentStore.Exists(pointer)
result.Exists, err = contentStore.Exists(pointer)
if err != nil {
_ = catFileBatchReader.CloseWithError(err)
break
}
if result.Exists {
if !result.InRepo {