foster/forgejo
1
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2024-11-24 08:57:03 -05:00
Code Issues Wiki Activity

Use native git variants by default with go-git variants as build tag (#13673)

Browse source 
* Move last commit cache back into modules/git

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Remove go-git from the interface for last commit cache

Signed-off-by: Andrew Thornton <art27@cantab.net>

* move cacheref to last_commit_cache

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Remove go-git from routers/private/hook

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Move FindLFSFiles to pipeline

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Make no-go-git variants

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Submodule RefID

Signed-off-by: Andrew Thornton <art27@cantab.net>

* fix issue with GetCommitsInfo

Signed-off-by: Andrew Thornton <art27@cantab.net>

* fix GetLastCommitForPaths

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Improve efficiency

Signed-off-by: Andrew Thornton <art27@cantab.net>

* More efficiency

Signed-off-by: Andrew Thornton <art27@cantab.net>

* even faster

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Reduce duplication

* As per @lunny

Signed-off-by: Andrew Thornton <art27@cantab.net>

* attempt to fix drone

Signed-off-by: Andrew Thornton <art27@cantab.net>

* fix test-tags

Signed-off-by: Andrew Thornton <art27@cantab.net>

* default to use no-go-git variants and add gogit build tag

Signed-off-by: Andrew Thornton <art27@cantab.net>

* placate lint

Signed-off-by: Andrew Thornton <art27@cantab.net>

* as per @6543

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
This commit is contained in:
zeripath 2020-12-17 14:00:47 +00:00 committed by GitHub
parent 0851a89581
commit 511f6138d4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
81 changed files with 3952 additions and 1492 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
.drone.yml
View file

@ -33,6 +33,16 @@ steps:
GOSUMDB: sum.golang.org
TAGS: bindata sqlite sqlite_unlock_notify
- name: lint-backend-gogit
pull: always
image: golang:1.15
commands:
- make lint-backend
environment:
GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
GOSUMDB: sum.golang.org
TAGS: bindata gogit sqlite sqlite_unlock_notify
- name: checks-frontend
image: node:14
commands:
@ -69,7 +79,7 @@ steps:
GOPROXY: off
GOOS: linux
GOARCH: arm64
TAGS: bindata
TAGS: bindata gogit
commands:
- make backend # test cross compile
- rm ./gitea # clean
@ -173,6 +183,17 @@ steps:
GITHUB_READ_TOKEN:
from_secret: github_read_token
- name: unit-test-gogit
pull: always
image: golang:1.15
commands:
- make unit-test-coverage test-check
environment:
GOPROXY: off
TAGS: bindata gogit sqlite sqlite_unlock_notify
GITHUB_READ_TOKEN:
from_secret: github_read_token
- name: test-mysql
image: golang:1.15
commands:
@ -305,7 +326,8 @@ steps:
- timeout -s ABRT 40m make test-sqlite-migration test-sqlite
environment:
GOPROXY: off
TAGS: bindata
TAGS: bindata gogit sqlite sqlite_unlock_notify
TEST_TAGS: gogit sqlite sqlite_unlock_notify
USE_REPO_TEST_DIR: 1
depends_on:
- build
@ -318,7 +340,8 @@ steps:
- timeout -s ABRT 40m make test-pgsql-migration test-pgsql
environment:
GOPROXY: off
TAGS: bindata
TAGS: bindata gogit
TEST_TAGS: gogit
TEST_LDAP: 1
USE_REPO_TEST_DIR: 1
depends_on:

19
Makefile
View file

@ -110,7 +110,10 @@ TAGS ?=
TAGS_SPLIT := $(subst $(COMMA), ,$(TAGS))
TAGS_EVIDENCE := $(MAKE_EVIDENCE_DIR)/tags
TEST_TAGS ?= sqlite sqlite_unlock_notify
GO_DIRS := cmd integrations models modules routers build services vendor tools
GO_SOURCES := $(wildcard *.go)
GO_SOURCES += $(shell find $(GO_DIRS) -type f -name "*.go" -not -path modules/options/bindata.go -not -path modules/public/bindata.go -not -path modules/templates/bindata.go)
@ -339,8 +342,8 @@ watch-backend: go-check
.PHONY: test
test:
@echo "Running go test..."
@$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='sqlite sqlite_unlock_notify' $(GO_PACKAGES)
@echo "Running go test with -tags '$(TEST_TAGS)'..."
@$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='$(TEST_TAGS)' $(GO_PACKAGES)
.PHONY: test-check
test-check:
@ -356,8 +359,8 @@ test-check:
.PHONY: test\#%
test\#%:
@echo "Running go test..."
@$(GO) test -mod=vendor -tags='sqlite sqlite_unlock_notify' -run $(subst .,/,$*) $(GO_PACKAGES)
@echo "Running go test with -tags '$(TEST_TAGS)'..."
@$(GO) test -mod=vendor -tags='$(TEST_TAGS)' -run $(subst .,/,$*) $(GO_PACKAGES)
.PHONY: coverage
coverage:
@ -365,8 +368,8 @@ coverage:
.PHONY: unit-test-coverage
unit-test-coverage:
@echo "Running unit-test-coverage..."
@$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='sqlite sqlite_unlock_notify' -cover -coverprofile coverage.out $(GO_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1
@echo "Running unit-test-coverage -tags '$(TEST_TAGS)'..."
@$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='$(TEST_TAGS)' -cover -coverprofile coverage.out $(GO_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1
.PHONY: vendor
vendor:
@ -511,7 +514,7 @@ integrations.mssql.test: git-check $(GO_SOURCES)
$(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.mssql.test
integrations.sqlite.test: git-check $(GO_SOURCES)
$(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.sqlite.test -tags 'sqlite sqlite_unlock_notify'
$(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.sqlite.test -tags '$(TEST_TAGS)'
integrations.cover.test: git-check $(GO_SOURCES)
$(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -coverpkg $(shell echo $(GO_PACKAGES) | tr ' ' ',') -o integrations.cover.test
@ -534,7 +537,7 @@ migrations.mssql.test: $(GO_SOURCES)
.PHONY: migrations.sqlite.test
migrations.sqlite.test: $(GO_SOURCES)
$(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/integrations/migration-test -o migrations.sqlite.test -tags 'sqlite sqlite_unlock_notify'
$(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/integrations/migration-test -o migrations.sqlite.test -tags '$(TEST_TAGS)'
.PHONY: check
check: test

1
docs/content/doc/installation/from-source.en-us.md
View file

@ -101,6 +101,7 @@ Depending on requirements, the following build tags can be included.
- `pam`: Enable support for PAM (Linux Pluggable Authentication Modules). Can
be used to authenticate local users or extend authentication to methods
available to PAM.
* `gogit`: (EXPERIMENTAL) Use go-git variants of git commands.
Bundling assets into the binary using the `bindata` build tag is recommended for
production deployments. It is possible to serve the static assets directly via a reverse proxy,

23
modules/cache/cache.go vendored
View file

@ -27,6 +27,24 @@ func newCache(cacheConfig setting.Cache) (mc.Cache, error) {
})
}
// Cache is the interface that operates the cache data.
type Cache interface {
// Put puts value into cache with key and expire time.
Put(key string, val interface{}, timeout int64) error
// Get gets cached value by given key.
Get(key string) interface{}
// Delete deletes cached value by given key.
Delete(key string) error
// Incr increases cached int-type value by given key as a counter.
Incr(key string) error
// Decr decreases cached int-type value by given key as a counter.
Decr(key string) error
// IsExist returns true if cached value exists.
IsExist(key string) bool
// Flush deletes all cached data.
Flush() error
}
// NewContext start cache service
func NewContext() error {
var err error
@ -40,6 +58,11 @@ func NewContext() error {
return err
}
// GetCache returns the currently configured cache
func GetCache() Cache {
return conn
}
// GetString returns the key value from cache with callback when no key exists in cache
func GetString(key string, getFunc func() (string, error)) (string, error) {
if conn == nil || setting.CacheService.TTL == 0 {

70
modules/cache/last_commit.go vendored
View file

@ -1,70 +0,0 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package cache
import (
"crypto/sha256"
"fmt"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
mc "gitea.com/macaron/cache"
"github.com/go-git/go-git/v5/plumbing/object"
)
// LastCommitCache represents a cache to store last commit
type LastCommitCache struct {
repoPath string
ttl int64
repo *git.Repository
commitCache map[string]*object.Commit
mc.Cache
}
// NewLastCommitCache creates a new last commit cache for repo
func NewLastCommitCache(repoPath string, gitRepo *git.Repository, ttl int64) *LastCommitCache {
return &LastCommitCache{
repoPath: repoPath,
repo: gitRepo,
commitCache: make(map[string]*object.Commit),
ttl: ttl,
Cache: conn,
}
}
func (c LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string {
hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, ref, entryPath)))
return fmt.Sprintf("last_commit:%x", hashBytes)
}
// Get get the last commit information by commit id and entry path
func (c LastCommitCache) Get(ref, entryPath string) (*object.Commit, error) {
v := c.Cache.Get(c.getCacheKey(c.repoPath, ref, entryPath))
if vs, ok := v.(string); ok {
log.Trace("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs)
if commit, ok := c.commitCache[vs]; ok {
log.Trace("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs)
return commit, nil
}
id, err := c.repo.ConvertToSHA1(vs)
if err != nil {
return nil, err
}
commit, err := c.repo.GoGitRepo().CommitObject(id)
if err != nil {
return nil, err
}
c.commitCache[vs] = commit
return commit, nil
}
return nil, nil
}
// Put put the last commit id with commit and entry path
func (c LastCommitCache) Put(ref, entryPath, commitID string) error {
log.Trace("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID)
return c.Cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl)
}

3
modules/convert/git_commit_test.go
View file

@ -13,7 +13,6 @@ import (
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/stretchr/testify/assert"
)
@ -21,7 +20,7 @@ func TestToCommitMeta(t *testing.T) {
assert.NoError(t, models.PrepareTestDatabase())
headRepo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
sha1, _ := git.NewIDFromString("0000000000000000000000000000000000000000")
signature := &object.Signature{Name: "Test Signature", Email: "test@email.com", When: time.Unix(0, 0)}
signature := &git.Signature{Name: "Test Signature", Email: "test@email.com", When: time.Unix(0, 0)}
tag := &git.Tag{
Name: "Test Tag",
ID: sha1,

243
modules/git/batch_reader_nogogit.go Normal file
View file

@ -0,0 +1,243 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bufio"
"bytes"
"math"
"strconv"
)
// ReadBatchLine reads the header line from cat-file --batch
// We expect:
// <sha> SP <type> SP <size> LF
func ReadBatchLine(rd *bufio.Reader) (sha []byte, typ string, size int64, err error) {
sha, err = rd.ReadBytes(' ')
if err != nil {
return
}
sha = sha[:len(sha)-1]
typ, err = rd.ReadString(' ')
if err != nil {
return
}
typ = typ[:len(typ)-1]
var sizeStr string
sizeStr, err = rd.ReadString('\n')
if err != nil {
return
}
size, err = strconv.ParseInt(sizeStr[:len(sizeStr)-1], 10, 64)
return
}
// ReadTagObjectID reads a tag object ID hash from a cat-file --batch stream, throwing away the rest of the stream.
func ReadTagObjectID(rd *bufio.Reader, size int64) (string, error) {
id := ""
var n int64
headerLoop:
for {
line, err := rd.ReadBytes('\n')
if err != nil {
return "", err
}
n += int64(len(line))
idx := bytes.Index(line, []byte{' '})
if idx < 0 {
continue
}
if string(line[:idx]) == "object" {
id = string(line[idx+1 : len(line)-1])
break headerLoop
}
}
// Discard the rest of the tag
discard := size - n
for discard > math.MaxInt32 {
_, err := rd.Discard(math.MaxInt32)
if err != nil {
return id, err
}
discard -= math.MaxInt32
}
_, err := rd.Discard(int(discard))
return id, err
}
// ReadTreeID reads a tree ID from a cat-file --batch stream, throwing away the rest of the stream.
func ReadTreeID(rd *bufio.Reader, size int64) (string, error) {
id := ""
var n int64
headerLoop:
for {
line, err := rd.ReadBytes('\n')
if err != nil {
return "", err
}
n += int64(len(line))
idx := bytes.Index(line, []byte{' '})
if idx < 0 {
continue
}
if string(line[:idx]) == "tree" {
id = string(line[idx+1 : len(line)-1])
break headerLoop
}
}
// Discard the rest of the commit
discard := size - n
for discard > math.MaxInt32 {
_, err := rd.Discard(math.MaxInt32)
if err != nil {
return id, err
}
discard -= math.MaxInt32
}
_, err := rd.Discard(int(discard))
return id, err
}
// git tree files are a list:
// <mode-in-ascii> SP <fname> NUL <20-byte SHA>
//
// Unfortunately this 20-byte notation is somewhat in conflict to all other git tools
// Therefore we need some method to convert these 20-byte SHAs to a 40-byte SHA
// constant hextable to help quickly convert between 20byte and 40byte hashes
const hextable = "0123456789abcdef"
// to40ByteSHA converts a 20-byte SHA in a 40-byte slice into a 40-byte sha in place
// without allocations. This is at least 100x quicker that hex.EncodeToString
// NB This requires that sha is a 40-byte slice
func to40ByteSHA(sha []byte) []byte {
for i := 19; i >= 0; i-- {
v := sha[i]
vhi, vlo := v>>4, v&0x0f
shi, slo := hextable[vhi], hextable[vlo]
sha[i*2], sha[i*2+1] = shi, slo
}
return sha
}
// ParseTreeLineSkipMode reads an entry from a tree in a cat-file --batch stream
// This simply skips the mode - saving a substantial amount of time and carefully avoids allocations - except where fnameBuf is too small.
// It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations
//
// Each line is composed of:
// <mode-in-ascii-dropping-initial-zeros> SP <fname> NUL <20-byte SHA>
//
// We don't attempt to convert the 20-byte SHA to 40-byte SHA to save a lot of time
func ParseTreeLineSkipMode(rd *bufio.Reader, fnameBuf, shaBuf []byte) (fname, sha []byte, n int, err error) {
var readBytes []byte
// Skip the Mode
readBytes, err = rd.ReadSlice(' ') // NB: DOES NOT ALLOCATE SIMPLY RETURNS SLICE WITHIN READER BUFFER
if err != nil {
return
}
n += len(readBytes)
// Deal with the fname
readBytes, err = rd.ReadSlice('\x00')
copy(fnameBuf, readBytes)
if len(fnameBuf) > len(readBytes) {
fnameBuf = fnameBuf[:len(readBytes)] // cut the buf the correct size
} else {
fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...) // extend the buf and copy in the missing bits
}
for err == bufio.ErrBufferFull { // Then we need to read more
readBytes, err = rd.ReadSlice('\x00')
fnameBuf = append(fnameBuf, readBytes...) // there is little point attempting to avoid allocations here so just extend
}
n += len(fnameBuf)
if err != nil {
return
}
fnameBuf = fnameBuf[:len(fnameBuf)-1] // Drop the terminal NUL
fname = fnameBuf // set the returnable fname to the slice
// Now deal with the 20-byte SHA
idx := 0
for idx < 20 {
read := 0
read, err = rd.Read(shaBuf[idx:20])
n += read
if err != nil {
return
}
idx += read
}
sha = shaBuf
return
}
// ParseTreeLine reads an entry from a tree in a cat-file --batch stream
// This carefully avoids allocations - except where fnameBuf is too small.
// It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations
//
// Each line is composed of:
// <mode-in-ascii-dropping-initial-zeros> SP <fname> NUL <20-byte SHA>
//
// We don't attempt to convert the 20-byte SHA to 40-byte SHA to save a lot of time
func ParseTreeLine(rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) {
var readBytes []byte
// Read the Mode
readBytes, err = rd.ReadSlice(' ')
if err != nil {
return
}
n += len(readBytes)
copy(modeBuf, readBytes)
if len(modeBuf) > len(readBytes) {
modeBuf = modeBuf[:len(readBytes)]
} else {
modeBuf = append(modeBuf, readBytes[len(modeBuf):]...)
}
mode = modeBuf[:len(modeBuf)-1] // Drop the SP
// Deal with the fname
readBytes, err = rd.ReadSlice('\x00')
copy(fnameBuf, readBytes)
if len(fnameBuf) > len(readBytes) {
fnameBuf = fnameBuf[:len(readBytes)]
} else {
fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...)
}
for err == bufio.ErrBufferFull {
readBytes, err = rd.ReadSlice('\x00')
fnameBuf = append(fnameBuf, readBytes...)
}
n += len(fnameBuf)
if err != nil {
return
}
fnameBuf = fnameBuf[:len(fnameBuf)-1]
fname = fnameBuf
// Deal with the 20-byte SHA
idx := 0
for idx < 20 {
read := 0
read, err = rd.Read(shaBuf[idx:20])
n += read
if err != nil {
return
}
idx += read
}
sha = shaBuf
return
}

21
modules/git/blob.go
View file

@ -10,28 +10,9 @@ import (
"encoding/base64"
"io"
"io/ioutil"
"github.com/go-git/go-git/v5/plumbing"
)
// Blob represents a Git object.
type Blob struct {
ID SHA1
gogitEncodedObj plumbing.EncodedObject
name string
}
// DataAsync gets a ReadCloser for the contents of a blob without reading it all.
// Calling the Close function on the result will discard all unread output.
func (b *Blob) DataAsync() (io.ReadCloser, error) {
return b.gogitEncodedObj.Reader()
}
// Size returns the uncompressed size of the blob
func (b *Blob) Size() int64 {
return b.gogitEncodedObj.Size()
}
// This file contains common functions between the gogit and !gogit variants for git Blobs
// Name returns name of the tree entry this blob object was created from (or empty string)
func (b *Blob) Name() string {

33
modules/git/blob_gogit.go Normal file
View file

@ -0,0 +1,33 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"io"
"github.com/go-git/go-git/v5/plumbing"
)
// Blob represents a Git object.
type Blob struct {
ID SHA1
gogitEncodedObj plumbing.EncodedObject
name string
}
// DataAsync gets a ReadCloser for the contents of a blob without reading it all.
// Calling the Close function on the result will discard all unread output.
func (b *Blob) DataAsync() (io.ReadCloser, error) {
return b.gogitEncodedObj.Reader()
}
// Size returns the uncompressed size of the blob
func (b *Blob) Size() int64 {
return b.gogitEncodedObj.Size()
}

77
modules/git/blob_nogogit.go Normal file
View file

@ -0,0 +1,77 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bufio"
"io"
"strconv"
"strings"
)
// Blob represents a Git object.
type Blob struct {
ID SHA1
gotSize bool
size int64
repoPath string
name string
}
// DataAsync gets a ReadCloser for the contents of a blob without reading it all.
// Calling the Close function on the result will discard all unread output.
func (b *Blob) DataAsync() (io.ReadCloser, error) {
stdoutReader, stdoutWriter := io.Pipe()
var err error
go func() {
stderr := &strings.Builder{}
err = NewCommand("cat-file", "--batch").RunInDirFullPipeline(b.repoPath, stdoutWriter, stderr, strings.NewReader(b.ID.String()+"\n"))
if err != nil {
err = ConcatenateError(err, stderr.String())
_ = stdoutWriter.CloseWithError(err)
} else {
_ = stdoutWriter.Close()
}
}()
bufReader := bufio.NewReader(stdoutReader)
_, _, size, err := ReadBatchLine(bufReader)
if err != nil {
stdoutReader.Close()
return nil, err
}
return &LimitedReaderCloser{
R: bufReader,
C: stdoutReader,
N: int64(size),
}, err
}
// Size returns the uncompressed size of the blob
func (b *Blob) Size() int64 {
if b.gotSize {
return b.size
}
size, err := NewCommand("cat-file", "-s", b.ID.String()).RunInDir(b.repoPath)
if err != nil {
log("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repoPath, err)
return 0
}
b.size, err = strconv.ParseInt(size[:len(size)-1], 10, 64)
if err != nil {
log("error whilst parsing size %s for %s in %s. Error: %v", size, b.ID.String(), b.repoPath, err)
return 0
}
b.gotSize = true
return b.size
}

13
modules/git/cache.go
View file

@ -1,13 +0,0 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package git
import "github.com/go-git/go-git/v5/plumbing/object"
// LastCommitCache cache
type LastCommitCache interface {
Get(ref, entryPath string) (*object.Commit, error)
Put(ref, entryPath, commitID string) error
}

2
modules/git/command.go
View file

@ -189,7 +189,7 @@ func (c *Command) RunInDirTimeoutEnv(env []string, timeout time.Duration, dir st
stdout := new(bytes.Buffer)
stderr := new(bytes.Buffer)
if err := c.RunInDirTimeoutEnvPipeline(env, timeout, dir, stdout, stderr); err != nil {
return nil, concatenateError(err, stderr.String())
return nil, ConcatenateError(err, stderr.String())
}
if stdout.Len() > 0 {

59
modules/git/commit.go
View file

@ -19,8 +19,6 @@ import (
"net/http"
"strconv"
"strings"
"github.com/go-git/go-git/v5/plumbing/object"
)
// Commit represents a git commit.
@ -43,61 +41,6 @@ type CommitGPGSignature struct {
Payload string //TODO check if can be reconstruct from the rest of commit information to not have duplicate data
}
func convertPGPSignature(c *object.Commit) *CommitGPGSignature {
if c.PGPSignature == "" {
return nil
}
var w strings.Builder
var err error
if _, err = fmt.Fprintf(&w, "tree %s\n", c.TreeHash.String()); err != nil {
return nil
}
for _, parent := range c.ParentHashes {
if _, err = fmt.Fprintf(&w, "parent %s\n", parent.String()); err != nil {
return nil
}
}
if _, err = fmt.Fprint(&w, "author "); err != nil {
return nil
}
if err = c.Author.Encode(&w); err != nil {
return nil
}
if _, err = fmt.Fprint(&w, "\ncommitter "); err != nil {
return nil
}
if err = c.Committer.Encode(&w); err != nil {
return nil
}
if _, err = fmt.Fprintf(&w, "\n\n%s", c.Message); err != nil {
return nil
}
return &CommitGPGSignature{
Signature: c.PGPSignature,
Payload: w.String(),
}
}
func convertCommit(c *object.Commit) *Commit {
return &Commit{
ID: c.Hash,
CommitMessage: c.Message,
Committer: &c.Committer,
Author: &c.Author,
Signature: convertPGPSignature(c),
Parents: c.ParentHashes,
}
}
// Message returns the commit message. Same as retrieving CommitMessage directly.
func (c *Commit) Message() string {
return c.CommitMessage
@ -576,7 +519,7 @@ func GetCommitFileStatus(repoPath, commitID string) (*CommitFileStatus, error) {
err := NewCommand("show", "--name-status", "--pretty=format:''", commitID).RunInDirPipeline(repoPath, w, stderr)
w.Close() // Close writer to exit parsing goroutine
if err != nil {
return nil, concatenateError(err, stderr.String())
return nil, ConcatenateError(err, stderr.String())
}
<-done

70
modules/git/commit_convert_gogit.go Normal file
View file

@ -0,0 +1,70 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"fmt"
"strings"
"github.com/go-git/go-git/v5/plumbing/object"
)
func convertPGPSignature(c *object.Commit) *CommitGPGSignature {
if c.PGPSignature == "" {
return nil
}
var w strings.Builder
var err error
if _, err = fmt.Fprintf(&w, "tree %s\n", c.TreeHash.String()); err != nil {
return nil
}
for _, parent := range c.ParentHashes {
if _, err = fmt.Fprintf(&w, "parent %s\n", parent.String()); err != nil {
return nil
}
}
if _, err = fmt.Fprint(&w, "author "); err != nil {
return nil
}
if err = c.Author.Encode(&w); err != nil {
return nil
}
if _, err = fmt.Fprint(&w, "\ncommitter "); err != nil {
return nil
}
if err = c.Committer.Encode(&w); err != nil {
return nil
}
if _, err = fmt.Fprintf(&w, "\n\n%s", c.Message); err != nil {
return nil
}
return &CommitGPGSignature{
Signature: c.PGPSignature,
Payload: w.String(),
}
}
func convertCommit(c *object.Commit) *Commit {
return &Commit{
ID: c.Hash,
CommitMessage: c.Message,
Committer: &c.Committer,
Author: &c.Author,
Signature: convertPGPSignature(c),
Parents: c.ParentHashes,
}
}

287
modules/git/commit_info.go
View file

@ -4,286 +4,9 @@
package git
import (
"path"
"github.com/emirpasic/gods/trees/binaryheap"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph"
)
// GetCommitsInfo gets information of all commits that are corresponding to these entries
func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache LastCommitCache) ([][]interface{}, *Commit, error) {
entryPaths := make([]string, len(tes)+1)
// Get the commit for the treePath itself
entryPaths[0] = ""
for i, entry := range tes {
entryPaths[i+1] = entry.Name()
}
commitNodeIndex, commitGraphFile := commit.repo.CommitNodeIndex()
if commitGraphFile != nil {
defer commitGraphFile.Close()
}
c, err := commitNodeIndex.Get(commit.ID)
if err != nil {
return nil, nil, err
}
var revs map[string]*object.Commit
if cache != nil {
var unHitPaths []string
revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache)
if err != nil {
return nil, nil, err
}
if len(unHitPaths) > 0 {
revs2, err := GetLastCommitForPaths(c, treePath, unHitPaths)
if err != nil {
return nil, nil, err
}
for k, v := range revs2 {
if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil {
return nil, nil, err
}
revs[k] = v
}
}
} else {
revs, err = GetLastCommitForPaths(c, treePath, entryPaths)
}
if err != nil {
return nil, nil, err
}
commit.repo.gogitStorage.Close()
commitsInfo := make([][]interface{}, len(tes))
for i, entry := range tes {
if rev, ok := revs[entry.Name()]; ok {
entryCommit := convertCommit(rev)
if entry.IsSubModule() {
subModuleURL := ""
var fullPath string
if len(treePath) > 0 {
fullPath = treePath + "/" + entry.Name()
} else {
fullPath = entry.Name()
}
if subModule, err := commit.GetSubModule(fullPath); err != nil {
return nil, nil, err
} else if subModule != nil {
subModuleURL = subModule.URL
}
subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String())
commitsInfo[i] = []interface{}{entry, subModuleFile}
} else {
commitsInfo[i] = []interface{}{entry, entryCommit}
}
} else {
commitsInfo[i] = []interface{}{entry, nil}
}
}
// Retrieve the commit for the treePath itself (see above). We basically
// get it for free during the tree traversal and it's used for listing
// pages to display information about newest commit for a given path.
var treeCommit *Commit
if treePath == "" {
treeCommit = commit
} else if rev, ok := revs[""]; ok {
treeCommit = convertCommit(rev)
treeCommit.repo = commit.repo
}
return commitsInfo, treeCommit, nil
}
type commitAndPaths struct {
commit cgobject.CommitNode
// Paths that are still on the branch represented by commit
paths []string
// Set of hashes for the paths
hashes map[string]plumbing.Hash
}
func getCommitTree(c cgobject.CommitNode, treePath string) (*object.Tree, error) {
tree, err := c.Tree()
if err != nil {
return nil, err
}
// Optimize deep traversals by focusing only on the specific tree
if treePath != "" {
tree, err = tree.Tree(treePath)
if err != nil {
return nil, err
}
}
return tree, nil
}
func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[string]plumbing.Hash, error) {
tree, err := getCommitTree(c, treePath)
if err == object.ErrDirectoryNotFound {
// The whole tree didn't exist, so return empty map
return make(map[string]plumbing.Hash), nil
}
if err != nil {
return nil, err
}
hashes := make(map[string]plumbing.Hash)
for _, path := range paths {
if path != "" {
entry, err := tree.FindEntry(path)
if err == nil {
hashes[path] = entry.Hash
}
} else {
hashes[path] = tree.Hash
}
}
return hashes, nil
}
func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache LastCommitCache) (map[string]*object.Commit, []string, error) {
var unHitEntryPaths []string
var results = make(map[string]*object.Commit)
for _, p := range paths {
lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
if err != nil {
return nil, nil, err
}
if lastCommit != nil {
results[p] = lastCommit
continue
}
unHitEntryPaths = append(unHitEntryPaths, p)
}
return results, unHitEntryPaths, nil
}
// GetLastCommitForPaths returns last commit information
func GetLastCommitForPaths(c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
// We do a tree traversal with nodes sorted by commit time
heap := binaryheap.NewWith(func(a, b interface{}) int {
if a.(*commitAndPaths).commit.CommitTime().Before(b.(*commitAndPaths).commit.CommitTime()) {
return 1
}
return -1
})
resultNodes := make(map[string]cgobject.CommitNode)
initialHashes, err := getFileHashes(c, treePath, paths)
if err != nil {
return nil, err
}
// Start search from the root commit and with full set of paths
heap.Push(&commitAndPaths{c, paths, initialHashes})
for {
cIn, ok := heap.Pop()
if !ok {
break
}
current := cIn.(*commitAndPaths)
// Load the parent commits for the one we are currently examining
numParents := current.commit.NumParents()
var parents []cgobject.CommitNode
for i := 0; i < numParents; i++ {
parent, err := current.commit.ParentNode(i)
if err != nil {
break
}
parents = append(parents, parent)
}
// Examine the current commit and set of interesting paths
pathUnchanged := make([]bool, len(current.paths))
parentHashes := make([]map[string]plumbing.Hash, len(parents))
for j, parent := range parents {
parentHashes[j], err = getFileHashes(parent, treePath, current.paths)
if err != nil {
break
}
for i, path := range current.paths {
if parentHashes[j][path] == current.hashes[path] {
pathUnchanged[i] = true
}
}
}
var remainingPaths []string
for i, path := range current.paths {
// The results could already contain some newer change for the same path,
// so don't override that and bail out on the file early.
if resultNodes[path] == nil {
if pathUnchanged[i] {
// The path existed with the same hash in at least one parent so it could
// not have been changed in this commit directly.
remainingPaths = append(remainingPaths, path)
} else {
// There are few possible cases how can we get here:
// - The path didn't exist in any parent, so it must have been created by
// this commit.
// - The path did exist in the parent commit, but the hash of the file has
// changed.
// - We are looking at a merge commit and the hash of the file doesn't
// match any of the hashes being merged. This is more common for directories,
// but it can also happen if a file is changed through conflict resolution.
resultNodes[path] = current.commit
}
}
}
if len(remainingPaths) > 0 {
// Add the parent nodes along with remaining paths to the heap for further
// processing.
for j, parent := range parents {
// Combine remainingPath with paths available on the parent branch
// and make union of them
remainingPathsForParent := make([]string, 0, len(remainingPaths))
newRemainingPaths := make([]string, 0, len(remainingPaths))
for _, path := range remainingPaths {
if parentHashes[j][path] == current.hashes[path] {
remainingPathsForParent = append(remainingPathsForParent, path)
} else {
newRemainingPaths = append(newRemainingPaths, path)
}
}
if remainingPathsForParent != nil {
heap.Push(&commitAndPaths{parent, remainingPathsForParent, parentHashes[j]})
}
if len(newRemainingPaths) == 0 {
break
} else {
remainingPaths = newRemainingPaths
}
}
}
}
// Post-processing
result := make(map[string]*object.Commit)
for path, commitNode := range resultNodes {
var err error
result[path], err = commitNode.Commit()
if err != nil {
return nil, err
}
}
return result, nil
// CommitInfo describes the first commit with the provided entry
type CommitInfo struct {
Entry *TreeEntry
Commit *Commit
SubModuleFile *SubModuleFile
}

291
modules/git/commit_info_gogit.go Normal file
View file

@ -0,0 +1,291 @@
// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"path"
"github.com/emirpasic/gods/trees/binaryheap"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph"
)
// GetCommitsInfo gets information of all commits that are corresponding to these entries
func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) {
entryPaths := make([]string, len(tes)+1)
// Get the commit for the treePath itself
entryPaths[0] = ""
for i, entry := range tes {
entryPaths[i+1] = entry.Name()
}
commitNodeIndex, commitGraphFile := commit.repo.CommitNodeIndex()
if commitGraphFile != nil {
defer commitGraphFile.Close()
}
c, err := commitNodeIndex.Get(commit.ID)
if err != nil {
return nil, nil, err
}
var revs map[string]*object.Commit
if cache != nil {
var unHitPaths []string
revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache)
if err != nil {
return nil, nil, err
}
if len(unHitPaths) > 0 {
revs2, err := GetLastCommitForPaths(c, treePath, unHitPaths)
if err != nil {
return nil, nil, err
}
for k, v := range revs2 {
if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil {
return nil, nil, err
}
revs[k] = v
}
}
} else {
revs, err = GetLastCommitForPaths(c, treePath, entryPaths)
}
if err != nil {
return nil, nil, err
}
commit.repo.gogitStorage.Close()
commitsInfo := make([]CommitInfo, len(tes))
for i, entry := range tes {
commitsInfo[i] = CommitInfo{
Entry: entry,
}
if rev, ok := revs[entry.Name()]; ok {
entryCommit := convertCommit(rev)
commitsInfo[i].Commit = entryCommit
if entry.IsSubModule() {
subModuleURL := ""
var fullPath string
if len(treePath) > 0 {
fullPath = treePath + "/" + entry.Name()
} else {
fullPath = entry.Name()
}
if subModule, err := commit.GetSubModule(fullPath); err != nil {
return nil, nil, err
} else if subModule != nil {
subModuleURL = subModule.URL
}
subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String())
commitsInfo[i].SubModuleFile = subModuleFile
}
}
}
// Retrieve the commit for the treePath itself (see above). We basically
// get it for free during the tree traversal and it's used for listing
// pages to display information about newest commit for a given path.
var treeCommit *Commit
if treePath == "" {
treeCommit = commit
} else if rev, ok := revs[""]; ok {
treeCommit = convertCommit(rev)
treeCommit.repo = commit.repo
}
return commitsInfo, treeCommit, nil
}
type commitAndPaths struct {
commit cgobject.CommitNode
// Paths that are still on the branch represented by commit
paths []string
// Set of hashes for the paths
hashes map[string]plumbing.Hash
}
func getCommitTree(c cgobject.CommitNode, treePath string) (*object.Tree, error) {
tree, err := c.Tree()
if err != nil {
return nil, err
}
// Optimize deep traversals by focusing only on the specific tree
if treePath != "" {
tree, err = tree.Tree(treePath)
if err != nil {
return nil, err
}
}
return tree, nil
}
func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[string]plumbing.Hash, error) {
tree, err := getCommitTree(c, treePath)
if err == object.ErrDirectoryNotFound {
// The whole tree didn't exist, so return empty map
return make(map[string]plumbing.Hash), nil
}
if err != nil {
return nil, err
}
hashes := make(map[string]plumbing.Hash)
for _, path := range paths {
if path != "" {
entry, err := tree.FindEntry(path)
if err == nil {
hashes[path] = entry.Hash
}
} else {
hashes[path] = tree.Hash
}
}
return hashes, nil
}
func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*object.Commit, []string, error) {
var unHitEntryPaths []string
var results = make(map[string]*object.Commit)
for _, p := range paths {
lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
if err != nil {
return nil, nil, err
}
if lastCommit != nil {
results[p] = lastCommit.(*object.Commit)
continue
}
unHitEntryPaths = append(unHitEntryPaths, p)
}
return results, unHitEntryPaths, nil
}
// GetLastCommitForPaths returns last commit information
func GetLastCommitForPaths(c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
// We do a tree traversal with nodes sorted by commit time
heap := binaryheap.NewWith(func(a, b interface{}) int {
if a.(*commitAndPaths).commit.CommitTime().Before(b.(*commitAndPaths).commit.CommitTime()) {
return 1
}
return -1
})
resultNodes := make(map[string]cgobject.CommitNode)
initialHashes, err := getFileHashes(c, treePath, paths)
if err != nil {
return nil, err
}
// Start search from the root commit and with full set of paths
heap.Push(&commitAndPaths{c, paths, initialHashes})
for {
cIn, ok := heap.Pop()
if !ok {
break
}
current := cIn.(*commitAndPaths)
// Load the parent commits for the one we are currently examining
numParents := current.commit.NumParents()
var parents []cgobject.CommitNode
for i := 0; i < numParents; i++ {
parent, err := current.commit.ParentNode(i)
if err != nil {
break
}
parents = append(parents, parent)
}
// Examine the current commit and set of interesting paths
pathUnchanged := make([]bool, len(current.paths))
parentHashes := make([]map[string]plumbing.Hash, len(parents))
for j, parent := range parents {
parentHashes[j], err = getFileHashes(parent, treePath, current.paths)
if err != nil {
break
}
for i, path := range current.paths {
if parentHashes[j][path] == current.hashes[path] {
pathUnchanged[i] = true
}
}
}
var remainingPaths []string
for i, path := range current.paths {
// The results could already contain some newer change for the same path,
// so don't override that and bail out on the file early.
if resultNodes[path] == nil {
if pathUnchanged[i] {
// The path existed with the same hash in at least one parent so it could
// not have been changed in this commit directly.
remainingPaths = append(remainingPaths, path)
} else {
// There are few possible cases how can we get here:
// - The path didn't exist in any parent, so it must have been created by
// this commit.
// - The path did exist in the parent commit, but the hash of the file has
// changed.
// - We are looking at a merge commit and the hash of the file doesn't
// match any of the hashes being merged. This is more common for directories,
// but it can also happen if a file is changed through conflict resolution.
resultNodes[path] = current.commit
}
}
}
if len(remainingPaths) > 0 {
// Add the parent nodes along with remaining paths to the heap for further
// processing.
for j, parent := range parents {
// Combine remainingPath with paths available on the parent branch
// and make union of them
remainingPathsForParent := make([]string, 0, len(remainingPaths))
newRemainingPaths := make([]string, 0, len(remainingPaths))
for _, path := range remainingPaths {
if parentHashes[j][path] == current.hashes[path] {
remainingPathsForParent = append(remainingPathsForParent, path)
} else {
newRemainingPaths = append(newRemainingPaths, path)
}
}
if remainingPathsForParent != nil {
heap.Push(&commitAndPaths{parent, remainingPathsForParent, parentHashes[j]})
}
if len(newRemainingPaths) == 0 {
break
} else {
remainingPaths = newRemainingPaths
}
}
}
}
// Post-processing
result := make(map[string]*object.Commit)
for path, commitNode := range resultNodes {
var err error
result[path], err = commitNode.Commit()
if err != nil {
return nil, err
}
}
return result, nil
}

370
modules/git/commit_info_nogogit.go Normal file
View file

@ -0,0 +1,370 @@
// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bufio"
"bytes"
"fmt"
"io"
"math"
"path"
"sort"
"strings"
)
// GetCommitsInfo gets information of all commits that are corresponding to these entries
func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) {
entryPaths := make([]string, len(tes)+1)
// Get the commit for the treePath itself
entryPaths[0] = ""
for i, entry := range tes {
entryPaths[i+1] = entry.Name()
}
var err error
var revs map[string]*Commit
if cache != nil {
var unHitPaths []string
revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache)
if err != nil {
return nil, nil, err
}
if len(unHitPaths) > 0 {
sort.Strings(unHitPaths)
commits, err := GetLastCommitForPaths(commit, treePath, unHitPaths)
if err != nil {
return nil, nil, err
}
for i, found := range commits {
if err := cache.Put(commit.ID.String(), path.Join(treePath, unHitPaths[i]), found.ID.String()); err != nil {
return nil, nil, err
}
revs[unHitPaths[i]] = found
}
}
} else {
sort.Strings(entryPaths)
revs = map[string]*Commit{}
var foundCommits []*Commit
foundCommits, err = GetLastCommitForPaths(commit, treePath, entryPaths)
for i, found := range foundCommits {
revs[entryPaths[i]] = found
}
}
if err != nil {
return nil, nil, err
}
commitsInfo := make([]CommitInfo, len(tes))
for i, entry := range tes {
commitsInfo[i] = CommitInfo{
Entry: entry,
}
if entryCommit, ok := revs[entry.Name()]; ok {
commitsInfo[i].Commit = entryCommit
if entry.IsSubModule() {
subModuleURL := ""
var fullPath string
if len(treePath) > 0 {
fullPath = treePath + "/" + entry.Name()
} else {
fullPath = entry.Name()
}
if subModule, err := commit.GetSubModule(fullPath); err != nil {
return nil, nil, err
} else if subModule != nil {
subModuleURL = subModule.URL
}
subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String())
commitsInfo[i].SubModuleFile = subModuleFile
}
}
}
// Retrieve the commit for the treePath itself (see above). We basically
// get it for free during the tree traversal and it's used for listing
// pages to display information about newest commit for a given path.
var treeCommit *Commit
var ok bool
if treePath == "" {
treeCommit = commit
} else if treeCommit, ok = revs[""]; ok {
treeCommit.repo = commit.repo
}
return commitsInfo, treeCommit, nil
}
func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*Commit, []string, error) {
var unHitEntryPaths []string
var results = make(map[string]*Commit)
for _, p := range paths {
lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
if err != nil {
return nil, nil, err
}
if lastCommit != nil {
results[p] = lastCommit.(*Commit)
continue
}
unHitEntryPaths = append(unHitEntryPaths, p)
}
return results, unHitEntryPaths, nil
}
// GetLastCommitForPaths returns last commit information
func GetLastCommitForPaths(commit *Commit, treePath string, paths []string) ([]*Commit, error) {
// We read backwards from the commit to obtain all of the commits
// We'll do this by using rev-list to provide us with parent commits in order
revListReader, revListWriter := io.Pipe()
defer func() {
_ = revListWriter.Close()
_ = revListReader.Close()
}()
go func() {
stderr := strings.Builder{}
err := NewCommand("rev-list", "--format=%T", commit.ID.String()).RunInDirPipeline(commit.repo.Path, revListWriter, &stderr)
if err != nil {
_ = revListWriter.CloseWithError(ConcatenateError(err, (&stderr).String()))
} else {
_ = revListWriter.Close()
}
}()
// We feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary.
// so let's create a batch stdin and stdout
batchStdinReader, batchStdinWriter := io.Pipe()
batchStdoutReader, batchStdoutWriter := io.Pipe()
defer func() {
_ = batchStdinReader.Close()
_ = batchStdinWriter.Close()
_ = batchStdoutReader.Close()
_ = batchStdoutWriter.Close()
}()
go func() {
stderr := strings.Builder{}
err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(commit.repo.Path, batchStdoutWriter, &stderr, batchStdinReader)
if err != nil {
_ = revListWriter.CloseWithError(ConcatenateError(err, (&stderr).String()))
} else {
_ = revListWriter.Close()
}
}()
// For simplicities sake we'll us a buffered reader
batchReader := bufio.NewReader(batchStdoutReader)
mapsize := 4096
if len(paths) > mapsize {
mapsize = len(paths)
}
path2idx := make(map[string]int, mapsize)
for i, path := range paths {
path2idx[path] = i
}
fnameBuf := make([]byte, 4096)
modeBuf := make([]byte, 40)
allShaBuf := make([]byte, (len(paths)+1)*20)
shaBuf := make([]byte, 20)
tmpTreeID := make([]byte, 40)
// commits is the returnable commits matching the paths provided
commits := make([]string, len(paths))
// ids are the blob/tree ids for the paths
ids := make([][]byte, len(paths))
// We'll use a scanner for the revList because it's simpler than a bufio.Reader
scan := bufio.NewScanner(revListReader)
revListLoop:
for scan.Scan() {
// Get the next parent commit ID
commitID := scan.Text()
if !scan.Scan() {
break revListLoop
}
commitID = commitID[7:]
rootTreeID := scan.Text()
// push the tree to the cat-file --batch process
_, err := batchStdinWriter.Write([]byte(rootTreeID + "\n"))
if err != nil {
return nil, err
}
currentPath := ""
// OK if the target tree path is "" and the "" is in the paths just set this now
if treePath == "" && paths[0] == "" {
// If this is the first time we see this set the id appropriate for this paths to this tree and set the last commit to curCommit
if len(ids[0]) == 0 {
ids[0] = []byte(rootTreeID)
commits[0] = string(commitID)
} else if bytes.Equal(ids[0], []byte(rootTreeID)) {
commits[0] = string(commitID)
}
}
treeReadingLoop:
for {
_, _, size, err := ReadBatchLine(batchReader)
if err != nil {
return nil, err
}
// Handle trees
// n is counter for file position in the tree file
var n int64
// Two options: currentPath is the targetTreepath
if treePath == currentPath {
// We are in the right directory
// Parse each tree line in turn. (don't care about mode here.)
for n < size {
fname, sha, count, err := ParseTreeLineSkipMode(batchReader, fnameBuf, shaBuf)
shaBuf = sha
if err != nil {
return nil, err
}
n += int64(count)
idx, ok := path2idx[string(fname)]
if ok {
// Now if this is the first time round set the initial Blob(ish) SHA ID and the commit
if len(ids[idx]) == 0 {
copy(allShaBuf[20*(idx+1):20*(idx+2)], shaBuf)
ids[idx] = allShaBuf[20*(idx+1) : 20*(idx+2)]
commits[idx] = string(commitID)
} else if bytes.Equal(ids[idx], shaBuf) {
commits[idx] = string(commitID)
}
}
// FIXME: is there any order to the way strings are emitted from cat-file?
// if there is - then we could skip once we've passed all of our data
}
break treeReadingLoop
}
var treeID []byte
// We're in the wrong directory
// Find target directory in this directory
idx := len(currentPath)
if idx > 0 {
idx++
}
target := strings.SplitN(treePath[idx:], "/", 2)[0]
for n < size {
// Read each tree entry in turn
mode, fname, sha, count, err := ParseTreeLine(batchReader, modeBuf, fnameBuf, shaBuf)
if err != nil {
return nil, err
}
n += int64(count)
// if we have found the target directory
if bytes.Equal(fname, []byte(target)) && bytes.Equal(mode, []byte("40000")) {
copy(tmpTreeID, sha)
treeID = tmpTreeID
break
}
}
if n < size {
// Discard any remaining entries in the current tree
discard := size - n
for discard > math.MaxInt32 {
_, err := batchReader.Discard(math.MaxInt32)
if err != nil {
return nil, err
}
discard -= math.MaxInt32
}
_, err := batchReader.Discard(int(discard))
if err != nil {
return nil, err
}
}
// if we haven't found a treeID for the target directory our search is over
if len(treeID) == 0 {
break treeReadingLoop
}
// add the target to the current path
if idx > 0 {
currentPath += "/"
}
currentPath += target
// if we've now found the current path check its sha id and commit status
if treePath == currentPath && paths[0] == "" {
if len(ids[0]) == 0 {
copy(allShaBuf[0:20], treeID)
ids[0] = allShaBuf[0:20]
commits[0] = string(commitID)
} else if bytes.Equal(ids[0], treeID) {
commits[0] = string(commitID)
}
}
treeID = to40ByteSHA(treeID)
_, err = batchStdinWriter.Write(treeID)
if err != nil {
return nil, err
}
_, err = batchStdinWriter.Write([]byte("\n"))
if err != nil {
return nil, err
}
}
}
commitsMap := make(map[string]*Commit, len(commits))
commitsMap[commit.ID.String()] = commit
commitCommits := make([]*Commit, len(commits))
for i, commitID := range commits {
c, ok := commitsMap[commitID]
if ok {
commitCommits[i] = c
continue
}
if len(commitID) == 0 {
continue
}
_, err := batchStdinWriter.Write([]byte(commitID + "\n"))
if err != nil {
return nil, err
}
_, typ, size, err := ReadBatchLine(batchReader)
if err != nil {
return nil, err
}
if typ != "commit" {
return nil, fmt.Errorf("unexpected type: %s for commit id: %s", typ, commitID)
}
c, err = CommitFromReader(commit.repo, MustIDFromString(string(commitID)), io.LimitReader(batchReader, int64(size)))
if err != nil {
return nil, err
}
commitCommits[i] = c
}
return commitCommits, scan.Err()
}

16
modules/git/commit_info_test.go
View file

@ -58,17 +58,27 @@ func testGetCommitsInfo(t *testing.T, repo1 *Repository) {
for _, testCase := range testCases {
commit, err := repo1.GetCommit(testCase.CommitID)
assert.NoError(t, err)
assert.NotNil(t, commit)
assert.NotNil(t, commit.Tree)
assert.NotNil(t, commit.Tree.repo)
tree, err := commit.Tree.SubTree(testCase.Path)
assert.NotNil(t, tree, "tree is nil for testCase CommitID %s in Path %s", testCase.CommitID, testCase.Path)
assert.NotNil(t, tree.repo, "repo is nil for testCase CommitID %s in Path %s", testCase.CommitID, testCase.Path)
assert.NoError(t, err)
entries, err := tree.ListEntries()
assert.NoError(t, err)
commitsInfo, treeCommit, err := entries.GetCommitsInfo(commit, testCase.Path, nil)
assert.Equal(t, testCase.ExpectedTreeCommit, treeCommit.ID.String())
assert.NoError(t, err)
if err != nil {
t.FailNow()
}
assert.Equal(t, testCase.ExpectedTreeCommit, treeCommit.ID.String())
assert.Len(t, commitsInfo, len(testCase.ExpectedIDs))
for _, commitInfo := range commitsInfo {
entry := commitInfo[0].(*TreeEntry)
commit := commitInfo[1].(*Commit)
entry := commitInfo.Entry
commit := commitInfo.Commit
expectedID, ok := testCase.ExpectedIDs[entry.Name()]
if !assert.True(t, ok) {
continue

46
modules/git/commit_reader.go
View file

@ -9,13 +9,13 @@ import (
"bytes"
"io"
"strings"
"github.com/go-git/go-git/v5/plumbing"
)
// CommitFromReader will generate a Commit from a provided reader
// We will need this to interpret commits from cat-file
func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader) (*Commit, error) {
// We need this to interpret commits from cat-file or cat-file --batch
//
// If used as part of a cat-file --batch stream you need to limit the reader to the correct size
func CommitFromReader(gitRepo *Repository, sha SHA1, reader io.Reader) (*Commit, error) {
commit := &Commit{
ID: sha,
}
@ -26,26 +26,20 @@ func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader)
message := false
pgpsig := false
scanner := bufio.NewScanner(reader)
// Split by '\n' but include the '\n'
scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
return 0, nil, nil
}
if i := bytes.IndexByte(data, '\n'); i >= 0 {
// We have a full newline-terminated line.
return i + 1, data[0 : i+1], nil
}
// If we're at EOF, we have a final, non-terminated line. Return it.
if atEOF {
return len(data), data, nil
}
// Request more data.
return 0, nil, nil
})
bufReader, ok := reader.(*bufio.Reader)
if !ok {
bufReader = bufio.NewReader(reader)
}
for scanner.Scan() {
line := scanner.Bytes()
readLoop:
for {
line, err := bufReader.ReadBytes('\n')
if err != nil {
if err == io.EOF {
break readLoop
}
return nil, err
}
if pgpsig {
if len(line) > 0 && line[0] == ' ' {
_, _ = signatureSB.Write(line[1:])
@ -72,10 +66,10 @@ func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader)
switch string(split[0]) {
case "tree":
commit.Tree = *NewTree(gitRepo, plumbing.NewHash(string(data)))
commit.Tree = *NewTree(gitRepo, MustIDFromString(string(data)))
_, _ = payloadSB.Write(line)
case "parent":
commit.Parents = append(commit.Parents, plumbing.NewHash(string(data)))
commit.Parents = append(commit.Parents, MustIDFromString(string(data)))
_, _ = payloadSB.Write(line)
case "author":
commit.Author = &Signature{}
@ -104,5 +98,5 @@ func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader)
commit.Signature = nil
}
return commit, scanner.Err()
return commit, nil
}

29
modules/git/last_commit_cache.go Normal file
View file

@ -0,0 +1,29 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package git
import (
"crypto/sha256"
"fmt"
)
// Cache represents a caching interface
type Cache interface {
// Put puts value into cache with key and expire time.
Put(key string, val interface{}, timeout int64) error
// Get gets cached value by given key.
Get(key string) interface{}
}
func (c *LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string {
hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, ref, entryPath)))
return fmt.Sprintf("last_commit:%x", hashBytes)
}
// Put put the last commit id with commit and entry path
func (c *LastCommitCache) Put(ref, entryPath, commitID string) error {
log("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID)
return c.cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl)
}

113
modules/git/last_commit_cache_gogit.go Normal file
View file

@ -0,0 +1,113 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"path"
"github.com/go-git/go-git/v5/plumbing/object"
cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph"
)
// LastCommitCache represents a cache to store last commit
type LastCommitCache struct {
repoPath string
ttl int64
repo *Repository
commitCache map[string]*object.Commit
cache Cache
}
// NewLastCommitCache creates a new last commit cache for repo
func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl int64, cache Cache) *LastCommitCache {
if cache == nil {
return nil
}
return &LastCommitCache{
repoPath: repoPath,
repo: gitRepo,
commitCache: make(map[string]*object.Commit),
ttl: ttl,
cache: cache,
}
}
// Get get the last commit information by commit id and entry path
func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) {
v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath))
if vs, ok := v.(string); ok {
log("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs)
if commit, ok := c.commitCache[vs]; ok {
log("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs)
return commit, nil
}
id, err := c.repo.ConvertToSHA1(vs)
if err != nil {
return nil, err
}
commit, err := c.repo.GoGitRepo().CommitObject(id)
if err != nil {
return nil, err
}
c.commitCache[vs] = commit
return commit, nil
}
return nil, nil
}
// CacheCommit will cache the commit from the gitRepository
func (c *LastCommitCache) CacheCommit(commit *Commit) error {
commitNodeIndex, _ := commit.repo.CommitNodeIndex()
index, err := commitNodeIndex.Get(commit.ID)
if err != nil {
return err
}
return c.recursiveCache(index, &commit.Tree, "", 1)
}
func (c *LastCommitCache) recursiveCache(index cgobject.CommitNode, tree *Tree, treePath string, level int) error {
if level == 0 {
return nil
}
entries, err := tree.ListEntries()
if err != nil {
return err
}
entryPaths := make([]string, len(entries))
entryMap := make(map[string]*TreeEntry)
for i, entry := range entries {
entryPaths[i] = entry.Name()
entryMap[entry.Name()] = entry
}
commits, err := GetLastCommitForPaths(index, treePath, entryPaths)
if err != nil {
return err
}
for entry, cm := range commits {
if err := c.Put(index.ID().String(), path.Join(treePath, entry), cm.ID().String()); err != nil {
return err
}
if entryMap[entry].IsDir() {
subTree, err := tree.SubTree(entry)
if err != nil {
return err
}
if err := c.recursiveCache(index, subTree, entry, level-1); err != nil {
return err
}
}
}
return nil
}

103
modules/git/last_commit_cache_nogogit.go Normal file
View file

@ -0,0 +1,103 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"path"
)
// LastCommitCache represents a cache to store last commit
type LastCommitCache struct {
repoPath string
ttl int64
repo *Repository
commitCache map[string]*Commit
cache Cache
}
// NewLastCommitCache creates a new last commit cache for repo
func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl int64, cache Cache) *LastCommitCache {
if cache == nil {
return nil
}
return &LastCommitCache{
repoPath: repoPath,
repo: gitRepo,
commitCache: make(map[string]*Commit),
ttl: ttl,
cache: cache,
}
}
// Get get the last commit information by commit id and entry path
func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) {
v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath))
if vs, ok := v.(string); ok {
log("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs)
if commit, ok := c.commitCache[vs]; ok {
log("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs)
return commit, nil
}
id, err := c.repo.ConvertToSHA1(vs)
if err != nil {
return nil, err
}
commit, err := c.repo.getCommit(id)
if err != nil {
return nil, err
}
c.commitCache[vs] = commit
return commit, nil
}
return nil, nil
}
// CacheCommit will cache the commit from the gitRepository
func (c *LastCommitCache) CacheCommit(commit *Commit) error {
return c.recursiveCache(commit, &commit.Tree, "", 1)
}
func (c *LastCommitCache) recursiveCache(commit *Commit, tree *Tree, treePath string, level int) error {
if level == 0 {
return nil
}
entries, err := tree.ListEntries()
if err != nil {
return err
}
entryPaths := make([]string, len(entries))
entryMap := make(map[string]*TreeEntry)
for i, entry := range entries {
entryPaths[i] = entry.Name()
entryMap[entry.Name()] = entry
}
commits, err := GetLastCommitForPaths(commit, treePath, entryPaths)
if err != nil {
return err
}
for i, entryCommit := range commits {
entry := entryPaths[i]
if err := c.Put(commit.ID.String(), path.Join(treePath, entryPaths[i]), entryCommit.ID.String()); err != nil {
return err
}
if entryMap[entry].IsDir() {
subTree, err := tree.SubTree(entry)
if err != nil {
return err
}
if err := c.recursiveCache(commit, subTree, entry, level-1); err != nil {
return err
}
}
}
return nil
}

65
modules/git/notes.go
View file

@ -4,12 +4,6 @@
package git
import (
"io/ioutil"
"github.com/go-git/go-git/v5/plumbing/object"
)
// NotesRef is the git ref where Gitea will look for git-notes data.
// The value ("refs/notes/commits") is the default ref used by git-notes.
const NotesRef = "refs/notes/commits"
@ -19,62 +13,3 @@ type Note struct {
Message []byte
Commit *Commit
}
// GetNote retrieves the git-notes data for a given commit.
func GetNote(repo *Repository, commitID string, note *Note) error {
notes, err := repo.GetCommit(NotesRef)
if err != nil {
return err
}
remainingCommitID := commitID
path := ""
currentTree := notes.Tree.gogitTree
var file *object.File
for len(remainingCommitID) > 2 {
file, err = currentTree.File(remainingCommitID)
if err == nil {
path += remainingCommitID
break
}
if err == object.ErrFileNotFound {
currentTree, err = currentTree.Tree(remainingCommitID[0:2])
path += remainingCommitID[0:2] + "/"
remainingCommitID = remainingCommitID[2:]
}
if err != nil {
return err
}
}
blob := file.Blob
dataRc, err := blob.Reader()
if err != nil {
return err
}
defer dataRc.Close()
d, err := ioutil.ReadAll(dataRc)
if err != nil {
return err
}
note.Message = d
commitNodeIndex, commitGraphFile := repo.CommitNodeIndex()
if commitGraphFile != nil {
defer commitGraphFile.Close()
}
commitNode, err := commitNodeIndex.Get(notes.ID)
if err != nil {
return err
}
lastCommits, err := GetLastCommitForPaths(commitNode, "", []string{path})
if err != nil {
return err
}
note.Commit = convertCommit(lastCommits[path])
return nil
}

72
modules/git/notes_gogit.go Normal file
View file

@ -0,0 +1,72 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"io/ioutil"
"github.com/go-git/go-git/v5/plumbing/object"
)
// GetNote retrieves the git-notes data for a given commit.
func GetNote(repo *Repository, commitID string, note *Note) error {
notes, err := repo.GetCommit(NotesRef)
if err != nil {
return err
}
remainingCommitID := commitID
path := ""
currentTree := notes.Tree.gogitTree
var file *object.File
for len(remainingCommitID) > 2 {
file, err = currentTree.File(remainingCommitID)
if err == nil {
path += remainingCommitID
break
}
if err == object.ErrFileNotFound {
currentTree, err = currentTree.Tree(remainingCommitID[0:2])
path += remainingCommitID[0:2] + "/"
remainingCommitID = remainingCommitID[2:]
}
if err != nil {
return err
}
}
blob := file.Blob
dataRc, err := blob.Reader()
if err != nil {
return err
}
defer dataRc.Close()
d, err := ioutil.ReadAll(dataRc)
if err != nil {
return err
}
note.Message = d
commitNodeIndex, commitGraphFile := repo.CommitNodeIndex()
if commitGraphFile != nil {
defer commitGraphFile.Close()
}
commitNode, err := commitNodeIndex.Get(notes.ID)
if err != nil {
return err
}
lastCommits, err := GetLastCommitForPaths(commitNode, "", []string{path})
if err != nil {
return err
}
note.Commit = convertCommit(lastCommits[path])
return nil
}

59
modules/git/notes_nogogit.go Normal file
View file

@ -0,0 +1,59 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"io/ioutil"
)
// GetNote retrieves the git-notes data for a given commit.
func GetNote(repo *Repository, commitID string, note *Note) error {
notes, err := repo.GetCommit(NotesRef)
if err != nil {
return err
}
path := ""
tree := &notes.Tree
var entry *TreeEntry
for len(commitID) > 2 {
entry, err = tree.GetTreeEntryByPath(commitID)
if err == nil {
path += commitID
break
}
if IsErrNotExist(err) {
tree, err = tree.SubTree(commitID[0:2])
path += commitID[0:2] + "/"
commitID = commitID[2:]
}
if err != nil {
return err
}
}
dataRc, err := entry.Blob().DataAsync()
if err != nil {
return err
}
defer dataRc.Close()
d, err := ioutil.ReadAll(dataRc)
if err != nil {
return err
}
note.Message = d
lastCommits, err := GetLastCommitForPaths(notes, "", []string{path})
if err != nil {
return err
}
note.Commit = lastCommits[0]
return nil
}

2
modules/git/parse.go → modules/git/parse_gogit.go
View file

@ -2,6 +2,8 @@
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (

2
modules/git/parse_test.go → modules/git/parse_gogit_test.go
View file

@ -2,6 +2,8 @@
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (

78
modules/git/parse_nogogit.go Normal file
View file

@ -0,0 +1,78 @@
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bytes"
"fmt"
"strconv"
)
// ParseTreeEntries parses the output of a `git ls-tree` command.
func ParseTreeEntries(data []byte) ([]*TreeEntry, error) {
return parseTreeEntries(data, nil)
}
func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) {
entries := make([]*TreeEntry, 0, 10)
for pos := 0; pos < len(data); {
// expect line to be of the form "<mode> <type> <sha>\t<filename>"
entry := new(TreeEntry)
entry.ptree = ptree
if pos+6 > len(data) {
return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data))
}
switch string(data[pos : pos+6]) {
case "100644":
entry.entryMode = EntryModeBlob
pos += 12 // skip over "100644 blob "
case "100755":
entry.entryMode = EntryModeExec
pos += 12 // skip over "100755 blob "
case "120000":
entry.entryMode = EntryModeSymlink
pos += 12 // skip over "120000 blob "
case "160000":
entry.entryMode = EntryModeCommit
pos += 14 // skip over "160000 object "
case "040000":
entry.entryMode = EntryModeTree
pos += 12 // skip over "040000 tree "
default:
return nil, fmt.Errorf("unknown type: %v", string(data[pos:pos+6]))
}
if pos+40 > len(data) {
return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data))
}
id, err := NewIDFromString(string(data[pos : pos+40]))
if err != nil {
return nil, fmt.Errorf("Invalid ls-tree output: %v", err)
}
entry.ID = id
pos += 41 // skip over sha and trailing space
end := pos + bytes.IndexByte(data[pos:], '\n')
if end < pos {
return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data))
}
// In case entry name is surrounded by double quotes(it happens only in git-shell).
if data[pos] == '"' {
entry.name, err = strconv.Unquote(string(data[pos:end]))
if err != nil {
return nil, fmt.Errorf("Invalid ls-tree output: %v", err)
}
} else {
entry.name = string(data[pos:end])
}
pos = end + 1
entries = append(entries, entry)
}
return entries, nil
}

159
modules/git/pipeline/lfs.go Normal file
View file

@ -0,0 +1,159 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package pipeline
import (
"bufio"
"fmt"
"io"
"sort"
"strings"
"sync"
"time"
"code.gitea.io/gitea/modules/git"
gogit "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/object"
)
// LFSResult represents commits found using a provided pointer file hash
type LFSResult struct {
Name string
SHA string
Summary string
When time.Time
ParentHashes []git.SHA1
BranchName string
FullCommitName string
}
type lfsResultSlice []*LFSResult
func (a lfsResultSlice) Len() int { return len(a) }
func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) }
// FindLFSFile finds commits that contain a provided pointer file hash
func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) {
resultsMap := map[string]*LFSResult{}
results := make([]*LFSResult, 0)
basePath := repo.Path
gogitRepo := repo.GoGitRepo()
commitsIter, err := gogitRepo.Log(&gogit.LogOptions{
Order: gogit.LogOrderCommitterTime,
All: true,
})
if err != nil {
return nil, fmt.Errorf("Failed to get GoGit CommitsIter. Error: %w", err)
}
err = commitsIter.ForEach(func(gitCommit *object.Commit) error {
tree, err := gitCommit.Tree()
if err != nil {
return err
}
treeWalker := object.NewTreeWalker(tree, true, nil)
defer treeWalker.Close()
for {
name, entry, err := treeWalker.Next()
if err == io.EOF {
break
}
if entry.Hash == hash {
result := LFSResult{
Name: name,
SHA: gitCommit.Hash.String(),
Summary: strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0],
When: gitCommit.Author.When,
ParentHashes: gitCommit.ParentHashes,
}
resultsMap[gitCommit.Hash.String()+":"+name] = &result
}
}
return nil
})
if err != nil && err != io.EOF {
return nil, fmt.Errorf("Failure in CommitIter.ForEach: %w", err)
}
for _, result := range resultsMap {
hasParent := false
for _, parentHash := range result.ParentHashes {
if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent {
break
}
}
if !hasParent {
results = append(results, result)
}
}
sort.Sort(lfsResultSlice(results))
// Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
shasToNameReader, shasToNameWriter := io.Pipe()
nameRevStdinReader, nameRevStdinWriter := io.Pipe()
errChan := make(chan error, 1)
wg := sync.WaitGroup{}
wg.Add(3)
go func() {
defer wg.Done()
scanner := bufio.NewScanner(nameRevStdinReader)
i := 0
for scanner.Scan() {
line := scanner.Text()
if len(line) == 0 {
continue
}
result := results[i]
result.FullCommitName = line
result.BranchName = strings.Split(line, "~")[0]
i++
}
}()
go NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath)
go func() {
defer wg.Done()
defer shasToNameWriter.Close()
for _, result := range results {
i := 0
if i < len(result.SHA) {
n, err := shasToNameWriter.Write([]byte(result.SHA)[i:])
if err != nil {
errChan <- err
break
}
i += n
}
n := 0
for n < 1 {
n, err = shasToNameWriter.Write([]byte{'\n'})
if err != nil {
errChan <- err
break
}
}
}
}()
wg.Wait()
select {
case err, has := <-errChan:
if has {
return nil, fmt.Errorf("Unable to obtain name for LFS files. Error: %w", err)
}
default:
}
return results, nil
}

266
modules/git/pipeline/lfs_nogogit.go Normal file
View file

@ -0,0 +1,266 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package pipeline
import (
"bufio"
"bytes"
"fmt"
"io"
"sort"
"strings"
"sync"
"time"
"code.gitea.io/gitea/modules/git"
)
// LFSResult represents commits found using a provided pointer file hash
type LFSResult struct {
Name string
SHA string
Summary string
When time.Time
ParentHashes []git.SHA1
BranchName string
FullCommitName string
}
type lfsResultSlice []*LFSResult
func (a lfsResultSlice) Len() int { return len(a) }
func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) }
// FindLFSFile finds commits that contain a provided pointer file hash
func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) {
resultsMap := map[string]*LFSResult{}
results := make([]*LFSResult, 0)
basePath := repo.Path
hashStr := hash.String()
// Use rev-list to provide us with all commits in order
revListReader, revListWriter := io.Pipe()
defer func() {
_ = revListWriter.Close()
_ = revListReader.Close()
}()
go func() {
stderr := strings.Builder{}
err := git.NewCommand("rev-list", "--all").RunInDirPipeline(repo.Path, revListWriter, &stderr)
if err != nil {
_ = revListWriter.CloseWithError(git.ConcatenateError(err, (&stderr).String()))
} else {
_ = revListWriter.Close()
}
}()
// Next feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary.
// so let's create a batch stdin and stdout
batchStdinReader, batchStdinWriter := io.Pipe()
batchStdoutReader, batchStdoutWriter := io.Pipe()
defer func() {
_ = batchStdinReader.Close()
_ = batchStdinWriter.Close()
_ = batchStdoutReader.Close()
_ = batchStdoutWriter.Close()
}()
go func() {
stderr := strings.Builder{}
err := git.NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, batchStdoutWriter, &stderr, batchStdinReader)
if err != nil {
_ = revListWriter.CloseWithError(git.ConcatenateError(err, (&stderr).String()))
} else {
_ = revListWriter.Close()
}
}()
// For simplicities sake we'll us a buffered reader to read from the cat-file --batch
batchReader := bufio.NewReader(batchStdoutReader)
// We'll use a scanner for the revList because it's simpler than a bufio.Reader
scan := bufio.NewScanner(revListReader)
trees := [][]byte{}
paths := []string{}
fnameBuf := make([]byte, 4096)
modeBuf := make([]byte, 40)
workingShaBuf := make([]byte, 40)
for scan.Scan() {
// Get the next commit ID
commitID := scan.Bytes()
// push the commit to the cat-file --batch process
_, err := batchStdinWriter.Write(commitID)
if err != nil {
return nil, err
}
_, err = batchStdinWriter.Write([]byte{'\n'})
if err != nil {
return nil, err
}
var curCommit *git.Commit
curPath := ""
commitReadingLoop:
for {
_, typ, size, err := git.ReadBatchLine(batchReader)
if err != nil {
return nil, err
}
switch typ {
case "tag":
// This shouldn't happen but if it does well just get the commit and try again
id, err := git.ReadTagObjectID(batchReader, size)
if err != nil {
return nil, err
}
_, err = batchStdinWriter.Write([]byte(id + "\n"))
if err != nil {
return nil, err
}
continue
case "commit":
// Read in the commit to get its tree and in case this is one of the last used commits
curCommit, err = git.CommitFromReader(repo, git.MustIDFromString(string(commitID)), io.LimitReader(batchReader, int64(size)))
if err != nil {
return nil, err
}
_, err := batchStdinWriter.Write([]byte(curCommit.Tree.ID.String() + "\n"))
if err != nil {
return nil, err
}
curPath = ""
case "tree":
var n int64
for n < size {
mode, fname, sha, count, err := git.ParseTreeLine(batchReader, modeBuf, fnameBuf, workingShaBuf)
if err != nil {
return nil, err
}
n += int64(count)
if bytes.Equal(sha, []byte(hashStr)) {
result := LFSResult{
Name: curPath + string(fname),
SHA: curCommit.ID.String(),
Summary: strings.Split(strings.TrimSpace(curCommit.CommitMessage), "\n")[0],
When: curCommit.Author.When,
ParentHashes: curCommit.Parents,
}
resultsMap[curCommit.ID.String()+":"+curPath+string(fname)] = &result
} else if string(mode) == git.EntryModeTree.String() {
trees = append(trees, sha)
paths = append(paths, curPath+string(fname)+"/")
}
}
if len(trees) > 0 {
_, err := batchStdinWriter.Write(trees[len(trees)-1])
if err != nil {
return nil, err
}
_, err = batchStdinWriter.Write([]byte("\n"))
if err != nil {
return nil, err
}
curPath = paths[len(paths)-1]
trees = trees[:len(trees)-1]
paths = paths[:len(paths)-1]
} else {
break commitReadingLoop
}
}
}
}
if err := scan.Err(); err != nil {
return nil, err
}
for _, result := range resultsMap {
hasParent := false
for _, parentHash := range result.ParentHashes {
if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent {
break
}
}
if !hasParent {
results = append(results, result)
}
}
sort.Sort(lfsResultSlice(results))
// Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
shasToNameReader, shasToNameWriter := io.Pipe()
nameRevStdinReader, nameRevStdinWriter := io.Pipe()
errChan := make(chan error, 1)
wg := sync.WaitGroup{}
wg.Add(3)
go func() {
defer wg.Done()
scanner := bufio.NewScanner(nameRevStdinReader)
i := 0
for scanner.Scan() {
line := scanner.Text()
if len(line) == 0 {
continue
}
result := results[i]
result.FullCommitName = line
result.BranchName = strings.Split(line, "~")[0]
i++
}
}()
go NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath)
go func() {
defer wg.Done()
defer shasToNameWriter.Close()
for _, result := range results {
i := 0
if i < len(result.SHA) {
n, err := shasToNameWriter.Write([]byte(result.SHA)[i:])
if err != nil {
errChan <- err
break
}
i += n
}
var err error
n := 0
for n < 1 {
n, err = shasToNameWriter.Write([]byte{'\n'})
if err != nil {
errChan <- err
break
}
}
}
}()
wg.Wait()
select {
case err, has := <-errChan:
if has {
return nil, fmt.Errorf("Unable to obtain name for LFS files. Error: %w", err)
}
default:
}
return results, nil
}

64
modules/git/repo.go
View file

@ -9,34 +9,16 @@ import (
"bytes"
"container/list"
"context"
"errors"
"fmt"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
gitealog "code.gitea.io/gitea/modules/log"
"github.com/go-git/go-billy/v5/osfs"
gogit "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/cache"
"github.com/go-git/go-git/v5/storage/filesystem"
"github.com/unknwon/com"
)
// Repository represents a Git repository.
type Repository struct {
Path string
tagCache *ObjectCache
gogitRepo *gogit.Repository
gogitStorage *filesystem.Storage
gpgSettings *GPGSettings
}
// GPGSettings represents the default GPG settings for this repository
type GPGSettings struct {
Sign bool
@ -93,52 +75,6 @@ func InitRepository(repoPath string, bare bool) error {
return err
}
// OpenRepository opens the repository at the given path.
func OpenRepository(repoPath string) (*Repository, error) {
repoPath, err := filepath.Abs(repoPath)
if err != nil {
return nil, err
} else if !isDir(repoPath) {
return nil, errors.New("no such file or directory")
}
fs := osfs.New(repoPath)
_, err = fs.Stat(".git")
if err == nil {
fs, err = fs.Chroot(".git")
if err != nil {
return nil, err
}
}
storage := filesystem.NewStorageWithOptions(fs, cache.NewObjectLRUDefault(), filesystem.Options{KeepDescriptors: true})
gogitRepo, err := gogit.Open(storage, fs)
if err != nil {
return nil, err
}
return &Repository{
Path: repoPath,
gogitRepo: gogitRepo,
gogitStorage: storage,
tagCache: newObjectCache(),
}, nil
}
// Close this repository, in particular close the underlying gogitStorage if this is not nil
func (repo *Repository) Close() {
if repo == nil || repo.gogitStorage == nil {
return
}
if err := repo.gogitStorage.Close(); err != nil {
gitealog.Error("Error closing storage: %v", err)
}
}
// GoGitRepo gets the go-git repo representation
func (repo *Repository) GoGitRepo() *gogit.Repository {
return repo.gogitRepo
}
// IsEmpty Check if repository is empty.
func (repo *Repository) IsEmpty() (bool, error) {
var errbuf strings.Builder

76
modules/git/repo_base_gogit.go Normal file
View file

@ -0,0 +1,76 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"errors"
"path/filepath"
gitealog "code.gitea.io/gitea/modules/log"
"github.com/go-git/go-billy/v5/osfs"
gogit "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/cache"
"github.com/go-git/go-git/v5/storage/filesystem"
)
// Repository represents a Git repository.
type Repository struct {
Path string
tagCache *ObjectCache
gogitRepo *gogit.Repository
gogitStorage *filesystem.Storage
gpgSettings *GPGSettings
}
// OpenRepository opens the repository at the given path.
func OpenRepository(repoPath string) (*Repository, error) {
repoPath, err := filepath.Abs(repoPath)
if err != nil {
return nil, err
} else if !isDir(repoPath) {
return nil, errors.New("no such file or directory")
}
fs := osfs.New(repoPath)
_, err = fs.Stat(".git")
if err == nil {
fs, err = fs.Chroot(".git")
if err != nil {
return nil, err
}
}
storage := filesystem.NewStorageWithOptions(fs, cache.NewObjectLRUDefault(), filesystem.Options{KeepDescriptors: true})
gogitRepo, err := gogit.Open(storage, fs)
if err != nil {
return nil, err
}
return &Repository{
Path: repoPath,
gogitRepo: gogitRepo,
gogitStorage: storage,
tagCache: newObjectCache(),
}, nil
}
// Close this repository, in particular close the underlying gogitStorage if this is not nil
func (repo *Repository) Close() {
if repo == nil || repo.gogitStorage == nil {
return
}
if err := repo.gogitStorage.Close(); err != nil {
gitealog.Error("Error closing storage: %v", err)
}
}
// GoGitRepo gets the go-git repo representation
func (repo *Repository) GoGitRepo() *gogit.Repository {
return repo.gogitRepo
}

40
modules/git/repo_base_nogogit.go Normal file
View file

@ -0,0 +1,40 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"errors"
"path/filepath"
)
// Repository represents a Git repository.
type Repository struct {
Path string
tagCache *ObjectCache
gpgSettings *GPGSettings
}
// OpenRepository opens the repository at the given path.
func OpenRepository(repoPath string) (*Repository, error) {
repoPath, err := filepath.Abs(repoPath)
if err != nil {
return nil, err
} else if !isDir(repoPath) {
return nil, errors.New("no such file or directory")
}
return &Repository{
Path: repoPath,
tagCache: newObjectCache(),
}, nil
}
// Close this repository, in particular close the underlying gogitStorage if this is not nil
func (repo *Repository) Close() {
}

18
modules/git/repo_blob.go
View file

@ -1,25 +1,9 @@
// Copyright 2018 The Gitea Authors. All rights reserved.
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package git
import (
"github.com/go-git/go-git/v5/plumbing"
)
func (repo *Repository) getBlob(id SHA1) (*Blob, error) {
encodedObj, err := repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, id)
if err != nil {
return nil, ErrNotExist{id.String(), ""}
}
return &Blob{
ID: id,
gogitEncodedObj: encodedObj,
}, nil
}
// GetBlob finds the blob object in the repository.
func (repo *Repository) GetBlob(idStr string) (*Blob, error) {
id, err := NewIDFromString(idStr)

23
modules/git/repo_blob_gogit.go Normal file
View file

@ -0,0 +1,23 @@
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"github.com/go-git/go-git/v5/plumbing"
)
func (repo *Repository) getBlob(id SHA1) (*Blob, error) {
encodedObj, err := repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, id)
if err != nil {
return nil, ErrNotExist{id.String(), ""}
}
return &Blob{
ID: id,
gogitEncodedObj: encodedObj,
}, nil
}

17
modules/git/repo_blob_nogogit.go Normal file
View file

@ -0,0 +1,17 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
func (repo *Repository) getBlob(id SHA1) (*Blob, error) {
if id.IsZero() {
return nil, ErrNotExist{id.String(), ""}
}
return &Blob{
ID: id,
repoPath: repo.Path,
}, nil
}

33
modules/git/repo_branch.go
View file

@ -8,8 +8,6 @@ package git
import (
"fmt"
"strings"
"github.com/go-git/go-git/v5/plumbing"
)
// BranchPrefix base dir of the branch information file store on git
@ -26,18 +24,6 @@ func IsBranchExist(repoPath, name string) bool {
return IsReferenceExist(repoPath, BranchPrefix+name)
}
// IsBranchExist returns true if given branch exists in current repository.
func (repo *Repository) IsBranchExist(name string) bool {
if name == "" {
return false
}
reference, err := repo.gogitRepo.Reference(plumbing.ReferenceName(BranchPrefix+name), true)
if err != nil {
return false
}
return reference.Type() != plumbing.InvalidReference
}
// Branch represents a Git branch.
type Branch struct {
Name string
@ -79,25 +65,6 @@ func (repo *Repository) GetDefaultBranch() (string, error) {
return NewCommand("symbolic-ref", "HEAD").RunInDir(repo.Path)
}
// GetBranches returns all branches of the repository.
func (repo *Repository) GetBranches() ([]string, error) {
var branchNames []string
branches, err := repo.gogitRepo.Branches()
if err != nil {
return nil, err
}
_ = branches.ForEach(func(branch *plumbing.Reference) error {
branchNames = append(branchNames, strings.TrimPrefix(branch.Name().String(), BranchPrefix))
return nil
})
// TODO: Sort?
return branchNames, nil
}
// GetBranch returns a branch by it's name
func (repo *Repository) GetBranch(branch string) (*Branch, error) {
if !repo.IsBranchExist(branch) {

45
modules/git/repo_branch_gogit.go Normal file
View file

@ -0,0 +1,45 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"strings"
"github.com/go-git/go-git/v5/plumbing"
)
// IsBranchExist returns true if given branch exists in current repository.
func (repo *Repository) IsBranchExist(name string) bool {
if name == "" {
return false
}
reference, err := repo.gogitRepo.Reference(plumbing.ReferenceName(BranchPrefix+name), true)
if err != nil {
return false
}
return reference.Type() != plumbing.InvalidReference
}
// GetBranches returns all branches of the repository.
func (repo *Repository) GetBranches() ([]string, error) {
var branchNames []string
branches, err := repo.gogitRepo.Branches()
if err != nil {
return nil, err
}
_ = branches.ForEach(func(branch *plumbing.Reference) error {
branchNames = append(branchNames, strings.TrimPrefix(branch.Name().String(), BranchPrefix))
return nil
})
// TODO: Sort?
return branchNames, nil
}

82
modules/git/repo_branch_nogogit.go Normal file
View file

@ -0,0 +1,82 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bufio"
"io"
"strings"
)
// IsBranchExist returns true if given branch exists in current repository.
func (repo *Repository) IsBranchExist(name string) bool {
if name == "" {
return false
}
return IsReferenceExist(repo.Path, BranchPrefix+name)
}
// GetBranches returns all branches of the repository.
func (repo *Repository) GetBranches() ([]string, error) {
return callShowRef(repo.Path, BranchPrefix, "--heads")
}
func callShowRef(repoPath, prefix, arg string) ([]string, error) {
var branchNames []string
stdoutReader, stdoutWriter := io.Pipe()
defer func() {
_ = stdoutReader.Close()
_ = stdoutWriter.Close()
}()
go func() {
stderrBuilder := &strings.Builder{}
err := NewCommand("show-ref", arg).RunInDirPipeline(repoPath, stdoutWriter, stderrBuilder)
if err != nil {
if stderrBuilder.Len() == 0 {
_ = stdoutWriter.Close()
return
}
_ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String()))
} else {
_ = stdoutWriter.Close()
}
}()
bufReader := bufio.NewReader(stdoutReader)
for {
// The output of show-ref is simply a list:
// <sha> SP <ref> LF
_, err := bufReader.ReadSlice(' ')
for err == bufio.ErrBufferFull {
// This shouldn't happen but we'll tolerate it for the sake of peace
_, err = bufReader.ReadSlice(' ')
}
if err == io.EOF {
return branchNames, nil
}
if err != nil {
return nil, err
}
branchName, err := bufReader.ReadString('\n')
if err == io.EOF {
// This shouldn't happen... but we'll tolerate it for the sake of peace
return branchNames, nil
}
if err != nil {
return nil, err
}
branchName = strings.TrimPrefix(branchName, prefix)
if len(branchName) > 0 {
branchName = branchName[:len(branchName)-1]
}
branchNames = append(branchNames, branchName)
}
}

98
modules/git/repo_commit.go
View file

@ -8,36 +8,10 @@ package git
import (
"bytes"
"container/list"
"fmt"
"strconv"
"strings"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
)
// GetRefCommitID returns the last commit ID string of given reference (branch or tag).
func (repo *Repository) GetRefCommitID(name string) (string, error) {
ref, err := repo.gogitRepo.Reference(plumbing.ReferenceName(name), true)
if err != nil {
if err == plumbing.ErrReferenceNotFound {
return "", ErrNotExist{
ID: name,
}
}
return "", err
}
return ref.Hash().String(), nil
}
// IsCommitExist returns true if given commit exists in current repository.
func (repo *Repository) IsCommitExist(name string) bool {
hash := plumbing.NewHash(name)
_, err := repo.gogitRepo.CommitObject(hash)
return err == nil
}
// GetBranchCommitID returns last commit ID string of given branch.
func (repo *Repository) GetBranchCommitID(name string) (string, error) {
return repo.GetRefCommitID(BranchPrefix + name)
@ -55,78 +29,6 @@ func (repo *Repository) GetTagCommitID(name string) (string, error) {
return strings.TrimSpace(stdout), nil
}
func convertPGPSignatureForTag(t *object.Tag) *CommitGPGSignature {
if t.PGPSignature == "" {
return nil
}
var w strings.Builder
var err error
if _, err = fmt.Fprintf(&w,
"object %s\ntype %s\ntag %s\ntagger ",
t.Target.String(), t.TargetType.Bytes(), t.Name); err != nil {
return nil
}
if err = t.Tagger.Encode(&w); err != nil {
return nil
}
if _, err = fmt.Fprintf(&w, "\n\n"); err != nil {
return nil
}
if _, err = fmt.Fprintf(&w, t.Message); err != nil {
return nil
}
return &CommitGPGSignature{
Signature: t.PGPSignature,
Payload: strings.TrimSpace(w.String()) + "\n",
}
}
func (repo *Repository) getCommit(id SHA1) (*Commit, error) {
var tagObject *object.Tag
gogitCommit, err := repo.gogitRepo.CommitObject(id)
if err == plumbing.ErrObjectNotFound {
tagObject, err = repo.gogitRepo.TagObject(id)
if err == plumbing.ErrObjectNotFound {
return nil, ErrNotExist{
ID: id.String(),
}
}
if err == nil {
gogitCommit, err = repo.gogitRepo.CommitObject(tagObject.Target)
}
// if we get a plumbing.ErrObjectNotFound here then the repository is broken and it should be 500
}
if err != nil {
return nil, err
}
commit := convertCommit(gogitCommit)
commit.repo = repo
if tagObject != nil {
commit.CommitMessage = strings.TrimSpace(tagObject.Message)
commit.Author = &tagObject.Tagger
commit.Signature = convertPGPSignatureForTag(tagObject)
}
tree, err := gogitCommit.Tree()
if err != nil {
return nil, err
}
commit.Tree.ID = tree.Hash
commit.Tree.gogitTree = tree
return commit, nil
}
// ConvertToSHA1 returns a Hash object from a potential ID string
func (repo *Repository) ConvertToSHA1(commitID string) (SHA1, error) {
if len(commitID) != 40 {

110
modules/git/repo_commit_gogit.go Normal file
View file

@ -0,0 +1,110 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"fmt"
"strings"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
)
// GetRefCommitID returns the last commit ID string of given reference (branch or tag).
func (repo *Repository) GetRefCommitID(name string) (string, error) {
ref, err := repo.gogitRepo.Reference(plumbing.ReferenceName(name), true)
if err != nil {
if err == plumbing.ErrReferenceNotFound {
return "", ErrNotExist{
ID: name,
}
}
return "", err
}
return ref.Hash().String(), nil
}
// IsCommitExist returns true if given commit exists in current repository.
func (repo *Repository) IsCommitExist(name string) bool {
hash := plumbing.NewHash(name)
_, err := repo.gogitRepo.CommitObject(hash)
return err == nil
}
func convertPGPSignatureForTag(t *object.Tag) *CommitGPGSignature {
if t.PGPSignature == "" {
return nil
}
var w strings.Builder
var err error
if _, err = fmt.Fprintf(&w,
"object %s\ntype %s\ntag %s\ntagger ",
t.Target.String(), t.TargetType.Bytes(), t.Name); err != nil {
return nil
}
if err = t.Tagger.Encode(&w); err != nil {
return nil
}
if _, err = fmt.Fprintf(&w, "\n\n"); err != nil {
return nil
}
if _, err = fmt.Fprintf(&w, t.Message); err != nil {
return nil
}
return &CommitGPGSignature{
Signature: t.PGPSignature,
Payload: strings.TrimSpace(w.String()) + "\n",
}
}
func (repo *Repository) getCommit(id SHA1) (*Commit, error) {
var tagObject *object.Tag
gogitCommit, err := repo.gogitRepo.CommitObject(id)
if err == plumbing.ErrObjectNotFound {
tagObject, err = repo.gogitRepo.TagObject(id)
if err == plumbing.ErrObjectNotFound {
return nil, ErrNotExist{
ID: id.String(),
}
}
if err == nil {
gogitCommit, err = repo.gogitRepo.CommitObject(tagObject.Target)
}
// if we get a plumbing.ErrObjectNotFound here then the repository is broken and it should be 500
}
if err != nil {
return nil, err
}
commit := convertCommit(gogitCommit)
commit.repo = repo
if tagObject != nil {
commit.CommitMessage = strings.TrimSpace(tagObject.Message)
commit.Author = &tagObject.Tagger
commit.Signature = convertPGPSignatureForTag(tagObject)
}
tree, err := gogitCommit.Tree()
if err != nil {
return nil, err
}
commit.Tree.ID = tree.Hash
commit.Tree.gogitTree = tree
return commit, nil
}

109
modules/git/repo_commit_nogogit.go Normal file
View file

@ -0,0 +1,109 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bufio"
"fmt"
"io"
"io/ioutil"
"strings"
)
// ResolveReference resolves a name to a reference
func (repo *Repository) ResolveReference(name string) (string, error) {
stdout, err := NewCommand("show-ref", "--hash", name).RunInDir(repo.Path)
if err != nil {
if strings.Contains(err.Error(), "not a valid ref") {
return "", ErrNotExist{name, ""}
}
return "", err
}
stdout = strings.TrimSpace(stdout)
if stdout == "" {
return "", ErrNotExist{name, ""}
}
return stdout, nil
}
// GetRefCommitID returns the last commit ID string of given reference (branch or tag).
func (repo *Repository) GetRefCommitID(name string) (string, error) {
stdout, err := NewCommand("show-ref", "--verify", "--hash", name).RunInDir(repo.Path)
if err != nil {
if strings.Contains(err.Error(), "not a valid ref") {
return "", ErrNotExist{name, ""}
}
return "", err
}
return strings.TrimSpace(stdout), nil
}
// IsCommitExist returns true if given commit exists in current repository.
func (repo *Repository) IsCommitExist(name string) bool {
_, err := NewCommand("cat-file", "-e", name).RunInDir(repo.Path)
return err == nil
}
func (repo *Repository) getCommit(id SHA1) (*Commit, error) {
stdoutReader, stdoutWriter := io.Pipe()
defer func() {
_ = stdoutReader.Close()
_ = stdoutWriter.Close()
}()
go func() {
stderr := strings.Builder{}
err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, stdoutWriter, &stderr, strings.NewReader(id.String()+"\n"))
if err != nil {
_ = stdoutWriter.CloseWithError(ConcatenateError(err, (&stderr).String()))
} else {
_ = stdoutWriter.Close()
}
}()
bufReader := bufio.NewReader(stdoutReader)
_, typ, size, err := ReadBatchLine(bufReader)
if err != nil {
return nil, err
}
switch typ {
case "tag":
// then we need to parse the tag
// and load the commit
data, err := ioutil.ReadAll(io.LimitReader(bufReader, size))
if err != nil {
return nil, err
}
tag, err := parseTagData(data)
if err != nil {
return nil, err
}
tag.repo = repo
commit, err := tag.Commit()
if err != nil {
return nil, err
}
commit.CommitMessage = strings.TrimSpace(tag.Message)
commit.Author = tag.Tagger
commit.Signature = tag.Signature
return commit, nil
case "commit":
return CommitFromReader(repo, id, io.LimitReader(bufReader, size))
default:
_ = stdoutReader.CloseWithError(fmt.Errorf("unknown typ: %s", typ))
log("Unknown typ: %s", typ)
return nil, ErrNotExist{
ID: id.String(),
}
}
}

2
modules/git/repo_commitgraph.go → modules/git/repo_commitgraph_gogit.go
View file

@ -3,6 +3,8 @@
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (

106
modules/git/repo_language_stats.go
View file

@ -4,111 +4,5 @@
package git
import (
"bytes"
"io"
"io/ioutil"
"code.gitea.io/gitea/modules/analyze"
"github.com/go-enry/go-enry/v2"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
)
const fileSizeLimit int64 = 16 * 1024 // 16 KiB
const bigFileSize int64 = 1024 * 1024 // 1 MiB
// GetLanguageStats calculates language stats for git repository at specified commit
func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) {
r, err := git.PlainOpen(repo.Path)
if err != nil {
return nil, err
}
rev, err := r.ResolveRevision(plumbing.Revision(commitID))
if err != nil {
return nil, err
}
commit, err := r.CommitObject(*rev)
if err != nil {
return nil, err
}
tree, err := commit.Tree()
if err != nil {
return nil, err
}
sizes := make(map[string]int64)
err = tree.Files().ForEach(func(f *object.File) error {
if f.Size == 0 || enry.IsVendor(f.Name) || enry.IsDotFile(f.Name) ||
enry.IsDocumentation(f.Name) || enry.IsConfiguration(f.Name) {
return nil
}
// If content can not be read or file is too big just do detection by filename
var content []byte
if f.Size <= bigFileSize {
content, _ = readFile(f, fileSizeLimit)
}
if enry.IsGenerated(f.Name, content) {
return nil
}
// TODO: Use .gitattributes file for linguist overrides
language := analyze.GetCodeLanguage(f.Name, content)
if language == enry.OtherLanguage || language == "" {
return nil
}
// group languages, such as Pug -> HTML; SCSS -> CSS
group := enry.GetLanguageGroup(language)
if group != "" {
language = group
}
sizes[language] += f.Size
return nil
})
if err != nil {
return nil, err
}
// filter special languages unless they are the only language
if len(sizes) > 1 {
for language := range sizes {
langtype := enry.GetLanguageType(language)
if langtype != enry.Programming && langtype != enry.Markup {
delete(sizes, language)
}
}
}
return sizes, nil
}
func readFile(f *object.File, limit int64) ([]byte, error) {
r, err := f.Reader()
if err != nil {
return nil, err
}
defer r.Close()
if limit <= 0 {
return ioutil.ReadAll(r)
}
size := f.Size
if limit > 0 && size > limit {
size = limit
}
buf := bytes.NewBuffer(nil)
buf.Grow(int(size))
_, err = io.Copy(buf, io.LimitReader(r, limit))
return buf.Bytes(), err
}

113
modules/git/repo_language_stats_gogit.go Normal file
View file

@ -0,0 +1,113 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"bytes"
"io"
"io/ioutil"
"code.gitea.io/gitea/modules/analyze"
"github.com/go-enry/go-enry/v2"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
)
// GetLanguageStats calculates language stats for git repository at specified commit
func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) {
r, err := git.PlainOpen(repo.Path)
if err != nil {
return nil, err
}
rev, err := r.ResolveRevision(plumbing.Revision(commitID))
if err != nil {
return nil, err
}
commit, err := r.CommitObject(*rev)
if err != nil {
return nil, err
}
tree, err := commit.Tree()
if err != nil {
return nil, err
}
sizes := make(map[string]int64)
err = tree.Files().ForEach(func(f *object.File) error {
if f.Size == 0 || enry.IsVendor(f.Name) || enry.IsDotFile(f.Name) ||
enry.IsDocumentation(f.Name) || enry.IsConfiguration(f.Name) {
return nil
}
// If content can not be read or file is too big just do detection by filename
var content []byte
if f.Size <= bigFileSize {
content, _ = readFile(f, fileSizeLimit)
}
if enry.IsGenerated(f.Name, content) {
return nil
}
// TODO: Use .gitattributes file for linguist overrides
language := analyze.GetCodeLanguage(f.Name, content)
if language == enry.OtherLanguage || language == "" {
return nil
}
// group languages, such as Pug -> HTML; SCSS -> CSS
group := enry.GetLanguageGroup(language)
if group != "" {
language = group
}
sizes[language] += f.Size
return nil
})
if err != nil {
return nil, err
}
// filter special languages unless they are the only language
if len(sizes) > 1 {
for language := range sizes {
langtype := enry.GetLanguageType(language)
if langtype != enry.Programming && langtype != enry.Markup {
delete(sizes, language)
}
}
}
return sizes, nil
}
func readFile(f *object.File, limit int64) ([]byte, error) {
r, err := f.Reader()
if err != nil {
return nil, err
}
defer r.Close()
if limit <= 0 {
return ioutil.ReadAll(r)
}
size := f.Size
if limit > 0 && size > limit {
size = limit
}
buf := bytes.NewBuffer(nil)
buf.Grow(int(size))
_, err = io.Copy(buf, io.LimitReader(r, limit))
return buf.Bytes(), err
}

109
modules/git/repo_language_stats_nogogit.go Normal file
View file

@ -0,0 +1,109 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bytes"
"io"
"io/ioutil"
"code.gitea.io/gitea/modules/analyze"
"github.com/go-enry/go-enry/v2"
)
// GetLanguageStats calculates language stats for git repository at specified commit
func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) {
// FIXME: We can be more efficient here...
//
// We're expecting that we will be reading a lot of blobs and the trees
// Thus we should use a shared `cat-file --batch` to get all of this data
// And keep the buffers around with resets as necessary.
//
// It's more complicated so...
commit, err := repo.GetCommit(commitID)
if err != nil {
log("Unable to get commit for: %s", commitID)
return nil, err
}
tree := commit.Tree
entries, err := tree.ListEntriesRecursive()
if err != nil {
return nil, err
}
sizes := make(map[string]int64)
for _, f := range entries {
if f.Size() == 0 || enry.IsVendor(f.Name()) || enry.IsDotFile(f.Name()) ||
enry.IsDocumentation(f.Name()) || enry.IsConfiguration(f.Name()) {
continue
}
// If content can not be read or file is too big just do detection by filename
var content []byte
if f.Size() <= bigFileSize {
content, _ = readFile(f, fileSizeLimit)
}
if enry.IsGenerated(f.Name(), content) {
continue
}
// TODO: Use .gitattributes file for linguist overrides
// FIXME: Why can't we split this and the IsGenerated tests to avoid reading the blob unless absolutely necessary?
// - eg. do the all the detection tests using filename first before reading content.
language := analyze.GetCodeLanguage(f.Name(), content)
if language == enry.OtherLanguage || language == "" {
continue
}
// group languages, such as Pug -> HTML; SCSS -> CSS
group := enry.GetLanguageGroup(language)
if group != "" {
language = group
}
sizes[language] += f.Size()
continue
}
// filter special languages unless they are the only language
if len(sizes) > 1 {
for language := range sizes {
langtype := enry.GetLanguageType(language)
if langtype != enry.Programming && langtype != enry.Markup {
delete(sizes, language)
}
}
}
return sizes, nil
}
func readFile(entry *TreeEntry, limit int64) ([]byte, error) {
// FIXME: We can probably be a little more efficient here... see above
r, err := entry.Blob().DataAsync()
if err != nil {
return nil, err
}
defer r.Close()
if limit <= 0 {
return ioutil.ReadAll(r)
}
size := entry.Size()
if limit > 0 && size > limit {
size = limit
}
buf := bytes.NewBuffer(nil)
buf.Grow(int(size))
_, err = io.Copy(buf, io.LimitReader(r, limit))
return buf.Bytes(), err
}

5
modules/git/repo_object.go
View file

@ -27,6 +27,11 @@ const (
ObjectBranch ObjectType = "branch"
)
// Bytes returns the byte array for the Object Type
func (o ObjectType) Bytes() []byte {
return []byte(o)
}
// HashObject takes a reader and returns SHA1 hash for that reader
func (repo *Repository) HashObject(reader io.Reader) (SHA1, error) {
idStr, err := repo.hashObject(reader)

45
modules/git/repo_ref.go
View file

@ -4,52 +4,7 @@
package git
import (
"strings"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
)
// GetRefs returns all references of the repository.
func (repo *Repository) GetRefs() ([]*Reference, error) {
return repo.GetRefsFiltered("")
}
// GetRefsFiltered returns all references of the repository that matches patterm exactly or starting with.
func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) {
r, err := git.PlainOpen(repo.Path)
if err != nil {
return nil, err
}
refsIter, err := r.References()
if err != nil {
return nil, err
}
refs := make([]*Reference, 0)
if err = refsIter.ForEach(func(ref *plumbing.Reference) error {
if ref.Name() != plumbing.HEAD && !ref.Name().IsRemote() &&
(pattern == "" || strings.HasPrefix(ref.Name().String(), pattern)) {
refType := string(ObjectCommit)
if ref.Name().IsTag() {
// tags can be of type `commit` (lightweight) or `tag` (annotated)
if tagType, _ := repo.GetTagType(ref.Hash()); err == nil {
refType = tagType
}
}
r := &Reference{
Name: ref.Name().String(),
Object: ref.Hash(),
Type: refType,
repo: repo,
}
refs = append(refs, r)
}
return nil
}); err != nil {
return nil, err
}
return refs, nil
}

52
modules/git/repo_ref_gogit.go Normal file
View file

@ -0,0 +1,52 @@
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"strings"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
)
// GetRefsFiltered returns all references of the repository that matches patterm exactly or starting with.
func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) {
r, err := git.PlainOpen(repo.Path)
if err != nil {
return nil, err
}
refsIter, err := r.References()
if err != nil {
return nil, err
}
refs := make([]*Reference, 0)
if err = refsIter.ForEach(func(ref *plumbing.Reference) error {
if ref.Name() != plumbing.HEAD && !ref.Name().IsRemote() &&
(pattern == "" || strings.HasPrefix(ref.Name().String(), pattern)) {
refType := string(ObjectCommit)
if ref.Name().IsTag() {
// tags can be of type `commit` (lightweight) or `tag` (annotated)
if tagType, _ := repo.GetTagType(ref.Hash()); err == nil {
refType = tagType
}
}
r := &Reference{
Name: ref.Name().String(),
Object: ref.Hash(),
Type: refType,
repo: repo,
}
refs = append(refs, r)
}
return nil
}); err != nil {
return nil, err
}
return refs, nil
}

84
modules/git/repo_ref_nogogit.go Normal file
View file

@ -0,0 +1,84 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bufio"
"io"
"strings"
)
// GetRefsFiltered returns all references of the repository that matches patterm exactly or starting with.
func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) {
stdoutReader, stdoutWriter := io.Pipe()
defer func() {
_ = stdoutReader.Close()
_ = stdoutWriter.Close()
}()
go func() {
stderrBuilder := &strings.Builder{}
err := NewCommand("for-each-ref").RunInDirPipeline(repo.Path, stdoutWriter, stderrBuilder)
if err != nil {
_ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String()))
} else {
_ = stdoutWriter.Close()
}
}()
refs := make([]*Reference, 0)
bufReader := bufio.NewReader(stdoutReader)
for {
// The output of for-each-ref is simply a list:
// <sha> SP <type> TAB <ref> LF
sha, err := bufReader.ReadString(' ')
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
sha = sha[:len(sha)-1]
typ, err := bufReader.ReadString('\t')
if err == io.EOF {
// This should not happen, but we'll tolerate it
break
}
if err != nil {
return nil, err
}
typ = typ[:len(typ)-1]
refName, err := bufReader.ReadString('\n')
if err == io.EOF {
// This should not happen, but we'll tolerate it
break
}
if err != nil {
return nil, err
}
refName = refName[:len(refName)-1]
// refName cannot be HEAD but can be remotes or stash
if strings.HasPrefix(refName, "/refs/remotes/") || refName == "/refs/stash" {
continue
}
if pattern == "" || strings.HasPrefix(refName, pattern) {
r := &Reference{
Name: refName,
Object: MustIDFromString(sha),
Type: typ,
repo: repo,
}
refs = append(refs, r)
}
}
return refs, nil
}

31
modules/git/repo_tag.go
View file

@ -8,8 +8,6 @@ package git
import (
"fmt"
"strings"
"github.com/go-git/go-git/v5/plumbing"
)
// TagPrefix tags prefix path on the repository
@ -20,12 +18,6 @@ func IsTagExist(repoPath, name string) bool {
return IsReferenceExist(repoPath, TagPrefix+name)
}
// IsTagExist returns true if given tag exists in the repository.
func (repo *Repository) IsTagExist(name string) bool {
_, err := repo.gogitRepo.Reference(plumbing.ReferenceName(TagPrefix+name), true)
return err == nil
}
// CreateTag create one tag in the repository
func (repo *Repository) CreateTag(name, revision string) error {
_, err := NewCommand("tag", "--", name, revision).RunInDir(repo.Path)
@ -224,29 +216,6 @@ func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, error) {
return tags, nil
}
// GetTags returns all tags of the repository.
func (repo *Repository) GetTags() ([]string, error) {
var tagNames []string
tags, err := repo.gogitRepo.Tags()
if err != nil {
return nil, err
}
_ = tags.ForEach(func(tag *plumbing.Reference) error {
tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix))
return nil
})
// Reverse order
for i := 0; i < len(tagNames)/2; i++ {
j := len(tagNames) - i - 1
tagNames[i], tagNames[j] = tagNames[j], tagNames[i]
}
return tagNames, nil
}
// GetTagType gets the type of the tag, either commit (simple) or tag (annotated)
func (repo *Repository) GetTagType(id SHA1) (string, error) {
// Get tag type

43
modules/git/repo_tag_gogit.go Normal file
View file

@ -0,0 +1,43 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"strings"
"github.com/go-git/go-git/v5/plumbing"
)
// IsTagExist returns true if given tag exists in the repository.
func (repo *Repository) IsTagExist(name string) bool {
_, err := repo.gogitRepo.Reference(plumbing.ReferenceName(TagPrefix+name), true)
return err == nil
}
// GetTags returns all tags of the repository.
func (repo *Repository) GetTags() ([]string, error) {
var tagNames []string
tags, err := repo.gogitRepo.Tags()
if err != nil {
return nil, err
}
_ = tags.ForEach(func(tag *plumbing.Reference) error {
tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix))
return nil
})
// Reverse order
for i := 0; i < len(tagNames)/2; i++ {
j := len(tagNames) - i - 1
tagNames[i], tagNames[j] = tagNames[j], tagNames[i]
}
return tagNames, nil
}

18
modules/git/repo_tag_nogogit.go Normal file
View file

@ -0,0 +1,18 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
// IsTagExist returns true if given tag exists in the repository.
func (repo *Repository) IsTagExist(name string) bool {
return IsReferenceExist(repo.Path, TagPrefix+name)
}
// GetTags returns all tags of the repository.
func (repo *Repository) GetTags() ([]string, error) {
return callShowRef(repo.Path, TagPrefix, "--tags")
}

41
modules/git/repo_tree.go
View file

@ -13,45 +13,6 @@ import (
"time"
)
func (repo *Repository) getTree(id SHA1) (*Tree, error) {
gogitTree, err := repo.gogitRepo.TreeObject(id)
if err != nil {
return nil, err
}
tree := NewTree(repo, id)
tree.gogitTree = gogitTree
return tree, nil
}
// GetTree find the tree object in the repository.
func (repo *Repository) GetTree(idStr string) (*Tree, error) {
if len(idStr) != 40 {
res, err := NewCommand("rev-parse", "--verify", idStr).RunInDir(repo.Path)
if err != nil {
return nil, err
}
if len(res) > 0 {
idStr = res[:len(res)-1]
}
}
id, err := NewIDFromString(idStr)
if err != nil {
return nil, err
}
resolvedID := id
commitObject, err := repo.gogitRepo.CommitObject(id)
if err == nil {
id = SHA1(commitObject.TreeHash)
}
treeObject, err := repo.getTree(id)
if err != nil {
return nil, err
}
treeObject.ResolvedID = resolvedID
return treeObject, nil
}
// CommitTreeOpts represents the possible options to CommitTree
type CommitTreeOpts struct {
Parents []string
@ -102,7 +63,7 @@ func (repo *Repository) CommitTree(author *Signature, committer *Signature, tree
err = cmd.RunInDirTimeoutEnvFullPipeline(env, -1, repo.Path, stdout, stderr, messageBytes)
if err != nil {
return SHA1{}, concatenateError(err, stderr.String())
return SHA1{}, ConcatenateError(err, stderr.String())
}
return NewIDFromString(strings.TrimSpace(stdout.String()))
}

47
modules/git/repo_tree_gogit.go Normal file
View file

@ -0,0 +1,47 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
func (repo *Repository) getTree(id SHA1) (*Tree, error) {
gogitTree, err := repo.gogitRepo.TreeObject(id)
if err != nil {
return nil, err
}
tree := NewTree(repo, id)
tree.gogitTree = gogitTree
return tree, nil
}
// GetTree find the tree object in the repository.
func (repo *Repository) GetTree(idStr string) (*Tree, error) {
if len(idStr) != 40 {
res, err := NewCommand("rev-parse", "--verify", idStr).RunInDir(repo.Path)
if err != nil {
return nil, err
}
if len(res) > 0 {
idStr = res[:len(res)-1]
}
}
id, err := NewIDFromString(idStr)
if err != nil {
return nil, err
}
resolvedID := id
commitObject, err := repo.gogitRepo.CommitObject(id)
if err == nil {
id = SHA1(commitObject.TreeHash)
}
treeObject, err := repo.getTree(id)
if err != nil {
return nil, err
}
treeObject.ResolvedID = resolvedID
return treeObject, nil
}

98
modules/git/repo_tree_nogogit.go Normal file
View file

@ -0,0 +1,98 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bufio"
"fmt"
"io"
"io/ioutil"
"strings"
)
func (repo *Repository) getTree(id SHA1) (*Tree, error) {
stdoutReader, stdoutWriter := io.Pipe()
defer func() {
_ = stdoutReader.Close()
_ = stdoutWriter.Close()
}()
go func() {
stderr := &strings.Builder{}
err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, stdoutWriter, stderr, strings.NewReader(id.String()+"\n"))
if err != nil {
_ = stdoutWriter.CloseWithError(ConcatenateError(err, stderr.String()))
} else {
_ = stdoutWriter.Close()
}
}()
bufReader := bufio.NewReader(stdoutReader)
// ignore the SHA
_, typ, _, err := ReadBatchLine(bufReader)
if err != nil {
return nil, err
}
switch typ {
case "tag":
resolvedID := id
data, err := ioutil.ReadAll(bufReader)
if err != nil {
return nil, err
}
tag, err := parseTagData(data)
if err != nil {
return nil, err
}
commit, err := tag.Commit()
if err != nil {
return nil, err
}
commit.Tree.ResolvedID = resolvedID
log("tag.commit.Tree: %s %v", commit.Tree.ID.String(), commit.Tree.repo)
return &commit.Tree, nil
case "commit":
commit, err := CommitFromReader(repo, id, bufReader)
if err != nil {
_ = stdoutReader.CloseWithError(err)
return nil, err
}
commit.Tree.ResolvedID = commit.ID
log("commit.Tree: %s %v", commit.Tree.ID.String(), commit.Tree.repo)
return &commit.Tree, nil
case "tree":
stdoutReader.Close()
tree := NewTree(repo, id)
tree.ResolvedID = id
return tree, nil
default:
_ = stdoutReader.CloseWithError(fmt.Errorf("unknown typ: %s", typ))
return nil, ErrNotExist{
ID: id.String(),
}
}
}
// GetTree find the tree object in the repository.
func (repo *Repository) GetTree(idStr string) (*Tree, error) {
if len(idStr) != 40 {
res, err := NewCommand("rev-parse", "--verify", idStr).RunInDir(repo.Path)
if err != nil {
return nil, err
}
if len(res) > 0 {
idStr = res[:len(res)-1]
}
}
id, err := NewIDFromString(idStr)
if err != nil {
return nil, err
}
return repo.getTree(id)
}

5
modules/git/sha1.go
View file

@ -10,8 +10,6 @@ import (
"fmt"
"regexp"
"strings"
"github.com/go-git/go-git/v5/plumbing"
)
// EmptySHA defines empty git SHA
@ -23,9 +21,6 @@ const EmptyTreeSHA = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
// SHAPattern can be used to determine if a string is an valid sha
var SHAPattern = regexp.MustCompile(`^[0-9a-f]{4,40}$`)
// SHA1 a git commit name
type SHA1 = plumbing.Hash
// MustID always creates a new SHA1 from a [20]byte array with no validation of input.
func MustID(b []byte) SHA1 {
var id SHA1

20
modules/git/sha1_gogit.go Normal file
View file

@ -0,0 +1,20 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"github.com/go-git/go-git/v5/plumbing"
)
// SHA1 a git commit name
type SHA1 = plumbing.Hash
// ComputeBlobHash compute the hash for a given blob content
func ComputeBlobHash(content []byte) SHA1 {
return plumbing.ComputeHash(plumbing.BlobObject, content)
}

62
modules/git/sha1_nogogit.go Normal file
View file

@ -0,0 +1,62 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"crypto/sha1"
"encoding/hex"
"hash"
"strconv"
)
// SHA1 a git commit name
type SHA1 [20]byte
// String returns a string representation of the SHA
func (s SHA1) String() string {
return hex.EncodeToString(s[:])
}
// IsZero returns whether this SHA1 is all zeroes
func (s SHA1) IsZero() bool {
var empty SHA1
return s == empty
}
// ComputeBlobHash compute the hash for a given blob content
func ComputeBlobHash(content []byte) SHA1 {
return ComputeHash(ObjectBlob, content)
}
// ComputeHash compute the hash for a given ObjectType and content
func ComputeHash(t ObjectType, content []byte) SHA1 {
h := NewHasher(t, int64(len(content)))
_, _ = h.Write(content)
return h.Sum()
}
// Hasher is a struct that will generate a SHA1
type Hasher struct {
hash.Hash
}
// NewHasher takes an object type and size and creates a hasher to generate a SHA
func NewHasher(t ObjectType, size int64) Hasher {
h := Hasher{sha1.New()}
_, _ = h.Write(t.Bytes())
_, _ = h.Write([]byte(" "))
_, _ = h.Write([]byte(strconv.FormatInt(size, 10)))
_, _ = h.Write([]byte{0})
return h
}
// Sum generates a SHA1 for the provided hash
func (h Hasher) Sum() (sha1 SHA1) {
copy(sha1[:], h.Hash.Sum(nil))
return
}

46
modules/git/signature.go
View file

@ -5,53 +5,7 @@
package git
import (
"bytes"
"strconv"
"time"
"github.com/go-git/go-git/v5/plumbing/object"
)
// Signature represents the Author or Committer information.
type Signature = object.Signature
const (
// GitTimeLayout is the (default) time layout used by git.
GitTimeLayout = "Mon Jan _2 15:04:05 2006 -0700"
)
// Helper to get a signature from the commit line, which looks like these:
// author Patrick Gundlach <gundlach@speedata.de> 1378823654 +0200
// author Patrick Gundlach <gundlach@speedata.de> Thu, 07 Apr 2005 22:13:13 +0200
// but without the "author " at the beginning (this method should)
// be used for author and committer.
//
// FIXME: include timezone for timestamp!
func newSignatureFromCommitline(line []byte) (_ *Signature, err error) {
sig := new(Signature)
emailStart := bytes.IndexByte(line, '<')
sig.Name = string(line[:emailStart-1])
emailEnd := bytes.IndexByte(line, '>')
sig.Email = string(line[emailStart+1 : emailEnd])
// Check date format.
if len(line) > emailEnd+2 {
firstChar := line[emailEnd+2]
if firstChar >= 48 && firstChar <= 57 {
timestop := bytes.IndexByte(line[emailEnd+2:], ' ')
timestring := string(line[emailEnd+2 : emailEnd+2+timestop])
seconds, _ := strconv.ParseInt(timestring, 10, 64)
sig.When = time.Unix(seconds, 0)
} else {
sig.When, err = time.Parse(GitTimeLayout, string(line[emailEnd+2:]))
if err != nil {
return nil, err
}
}
} else {
// Fall back to unix 0 time
sig.When = time.Unix(0, 0)
}
return sig, nil
}

54
modules/git/signature_gogit.go Normal file
View file

@ -0,0 +1,54 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"bytes"
"strconv"
"time"
"github.com/go-git/go-git/v5/plumbing/object"
)
// Signature represents the Author or Committer information.
type Signature = object.Signature
// Helper to get a signature from the commit line, which looks like these:
// author Patrick Gundlach <gundlach@speedata.de> 1378823654 +0200
// author Patrick Gundlach <gundlach@speedata.de> Thu, 07 Apr 2005 22:13:13 +0200
// but without the "author " at the beginning (this method should)
// be used for author and committer.
//
// FIXME: include timezone for timestamp!
func newSignatureFromCommitline(line []byte) (_ *Signature, err error) {
sig := new(Signature)
emailStart := bytes.IndexByte(line, '<')
sig.Name = string(line[:emailStart-1])
emailEnd := bytes.IndexByte(line, '>')
sig.Email = string(line[emailStart+1 : emailEnd])
// Check date format.
if len(line) > emailEnd+2 {
firstChar := line[emailEnd+2]
if firstChar >= 48 && firstChar <= 57 {
timestop := bytes.IndexByte(line[emailEnd+2:], ' ')
timestring := string(line[emailEnd+2 : emailEnd+2+timestop])
seconds, _ := strconv.ParseInt(timestring, 10, 64)
sig.When = time.Unix(seconds, 0)
} else {
sig.When, err = time.Parse(GitTimeLayout, string(line[emailEnd+2:]))
if err != nil {
return nil, err
}
}
} else {
// Fall back to unix 0 time
sig.When = time.Unix(0, 0)
}
return sig, nil
}

95
modules/git/signature_nogogit.go Normal file
View file

@ -0,0 +1,95 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bytes"
"fmt"
"strconv"
"time"
)
// Signature represents the Author or Committer information.
type Signature struct {
// Name represents a person name. It is an arbitrary string.
Name string
// Email is an email, but it cannot be assumed to be well-formed.
Email string
// When is the timestamp of the signature.
When time.Time
}
func (s *Signature) String() string {
return fmt.Sprintf("%s <%s>", s.Name, s.Email)
}
// Decode decodes a byte array representing a signature to signature
func (s *Signature) Decode(b []byte) {
sig, _ := newSignatureFromCommitline(b)
s.Email = sig.Email
s.Name = sig.Name
s.When = sig.When
}
// Helper to get a signature from the commit line, which looks like these:
// author Patrick Gundlach <gundlach@speedata.de> 1378823654 +0200
// author Patrick Gundlach <gundlach@speedata.de> Thu, 07 Apr 2005 22:13:13 +0200
// but without the "author " at the beginning (this method should)
// be used for author and committer.
func newSignatureFromCommitline(line []byte) (sig *Signature, err error) {
sig = new(Signature)
emailStart := bytes.LastIndexByte(line, '<')
emailEnd := bytes.LastIndexByte(line, '>')
if emailStart == -1 || emailEnd == -1 || emailEnd < emailStart {
return
}
sig.Name = string(line[:emailStart-1])
sig.Email = string(line[emailStart+1 : emailEnd])
hasTime := emailEnd+2 < len(line)
if !hasTime {
return
}
// Check date format.
firstChar := line[emailEnd+2]
if firstChar >= 48 && firstChar <= 57 {
idx := bytes.IndexByte(line[emailEnd+2:], ' ')
if idx < 0 {
return
}
timestring := string(line[emailEnd+2 : emailEnd+2+idx])
seconds, _ := strconv.ParseInt(timestring, 10, 64)
sig.When = time.Unix(seconds, 0)
idx += emailEnd + 3
if idx >= len(line) || idx+5 > len(line) {
return
}
timezone := string(line[idx : idx+5])
tzhours, err1 := strconv.ParseInt(timezone[0:3], 10, 64)
tzmins, err2 := strconv.ParseInt(timezone[3:], 10, 64)
if err1 != nil || err2 != nil {
return
}
if tzhours < 0 {
tzmins *= -1
}
tz := time.FixedZone("", int(tzhours*60*60+tzmins*60))
sig.When = sig.When.In(tz)
} else {
sig.When, err = time.Parse(GitTimeLayout, string(line[emailEnd+2:]))
if err != nil {
return
}
}
return
}

31
modules/git/tag.go
View file

@ -10,15 +10,19 @@ import (
"strings"
)
const beginpgp = "\n-----BEGIN PGP SIGNATURE-----\n"
const endpgp = "\n-----END PGP SIGNATURE-----"
// Tag represents a Git tag.
type Tag struct {
Name string
ID SHA1
repo *Repository
Object SHA1 // The id of this commit object
Type string
Tagger *Signature
Message string
Name string
ID SHA1
repo *Repository
Object SHA1 // The id of this commit object
Type string
Tagger *Signature
Message string
Signature *CommitGPGSignature
}
// Commit return the commit of the tag reference
@ -60,12 +64,23 @@ l:
}
nextline += eol + 1
case eol == 0:
tag.Message = strings.TrimRight(string(data[nextline+1:]), "\n")
tag.Message = string(data[nextline+1 : len(data)-1])
break l
default:
break l
}
}
idx := strings.LastIndex(tag.Message, beginpgp)
if idx > 0 {
endSigIdx := strings.Index(tag.Message[idx:], endpgp)
if endSigIdx > 0 {
tag.Signature = &CommitGPGSignature{
Signature: tag.Message[idx+1 : idx+endSigIdx+len(endpgp)],
Payload: string(data[:bytes.LastIndex(data, []byte(beginpgp))+1]),
}
tag.Message = tag.Message[:idx+1]
}
}
return tag, nil
}

83
modules/git/tree.go
View file

@ -6,25 +6,9 @@
package git
import (
"io"
"strings"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
)
// Tree represents a flat directory listing.
type Tree struct {
ID SHA1
ResolvedID SHA1
repo *Repository
gogitTree *object.Tree
// parent tree
ptree *Tree
}
// NewTree create a new tree according the repository and tree id
func NewTree(repo *Repository, id SHA1) *Tree {
return &Tree{
@ -61,70 +45,3 @@ func (t *Tree) SubTree(rpath string) (*Tree, error) {
}
return g, nil
}
func (t *Tree) loadTreeObject() error {
gogitTree, err := t.repo.gogitRepo.TreeObject(t.ID)
if err != nil {
return err
}
t.gogitTree = gogitTree
return nil
}
// ListEntries returns all entries of current tree.
func (t *Tree) ListEntries() (Entries, error) {
if t.gogitTree == nil {
err := t.loadTreeObject()
if err != nil {
return nil, err
}
}
entries := make([]*TreeEntry, len(t.gogitTree.Entries))
for i, entry := range t.gogitTree.Entries {
entries[i] = &TreeEntry{
ID: entry.Hash,
gogitTreeEntry: &t.gogitTree.Entries[i],
ptree: t,
}
}
return entries, nil
}
// ListEntriesRecursive returns all entries of current tree recursively including all subtrees
func (t *Tree) ListEntriesRecursive() (Entries, error) {
if t.gogitTree == nil {
err := t.loadTreeObject()
if err != nil {
return nil, err
}
}
var entries []*TreeEntry
seen := map[plumbing.Hash]bool{}
walker := object.NewTreeWalker(t.gogitTree, true, seen)
for {
fullName, entry, err := walker.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
if seen[entry.Hash] {
continue
}
convertedEntry := &TreeEntry{
ID: entry.Hash,
gogitTreeEntry: &entry,
ptree: t,
fullName: fullName,
}
entries = append(entries, convertedEntry)
}
return entries, nil
}

58
modules/git/tree_blob.go
View file

@ -5,64 +5,6 @@
package git
import (
"path"
"strings"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/object"
)
// GetTreeEntryByPath get the tree entries according the sub dir
func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) {
if len(relpath) == 0 {
return &TreeEntry{
ID: t.ID,
//Type: ObjectTree,
gogitTreeEntry: &object.TreeEntry{
Name: "",
Mode: filemode.Dir,
Hash: t.ID,
},
}, nil
}
relpath = path.Clean(relpath)
parts := strings.Split(relpath, "/")
var err error
tree := t
for i, name := range parts {
if i == len(parts)-1 {
entries, err := tree.ListEntries()
if err != nil {
if err == plumbing.ErrObjectNotFound {
return nil, ErrNotExist{
RelPath: relpath,
}
}
return nil, err
}
for _, v := range entries {
if v.Name() == name {
return v, nil
}
}
} else {
tree, err = tree.SubTree(name)
if err != nil {
if err == plumbing.ErrObjectNotFound {
return nil, ErrNotExist{
RelPath: relpath,
}
}
return nil, err
}
}
}
return nil, ErrNotExist{"", relpath}
}
// GetBlobByPath get the blob object according the path
func (t *Tree) GetBlobByPath(relpath string) (*Blob, error) {
entry, err := t.GetTreeEntryByPath(relpath)

66
modules/git/tree_blob_gogit.go Normal file
View file

@ -0,0 +1,66 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"path"
"strings"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/object"
)
// GetTreeEntryByPath get the tree entries according the sub dir
func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) {
if len(relpath) == 0 {
return &TreeEntry{
ID: t.ID,
//Type: ObjectTree,
gogitTreeEntry: &object.TreeEntry{
Name: "",
Mode: filemode.Dir,
Hash: t.ID,
},
}, nil
}
relpath = path.Clean(relpath)
parts := strings.Split(relpath, "/")
var err error
tree := t
for i, name := range parts {
if i == len(parts)-1 {
entries, err := tree.ListEntries()
if err != nil {
if err == plumbing.ErrObjectNotFound {
return nil, ErrNotExist{
RelPath: relpath,
}
}
return nil, err
}
for _, v := range entries {
if v.Name() == name {
return v, nil
}
}
} else {
tree, err = tree.SubTree(name)
if err != nil {
if err == plumbing.ErrObjectNotFound {
return nil, ErrNotExist{
RelPath: relpath,
}
}
return nil, err
}
}
}
return nil, ErrNotExist{"", relpath}
}

49
modules/git/tree_blob_nogogit.go Normal file
View file

@ -0,0 +1,49 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"path"
"strings"
)
// GetTreeEntryByPath get the tree entries according the sub dir
func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) {
if len(relpath) == 0 {
return &TreeEntry{
ID: t.ID,
name: "",
fullName: "",
entryMode: EntryModeTree,
}, nil
}
// FIXME: This should probably use git cat-file --batch to be a bit more efficient
relpath = path.Clean(relpath)
parts := strings.Split(relpath, "/")
var err error
tree := t
for i, name := range parts {
if i == len(parts)-1 {
entries, err := tree.ListEntries()
if err != nil {
return nil, err
}
for _, v := range entries {
if v.Name() == name {
return v, nil
}
}
} else {
tree, err = tree.SubTree(name)
if err != nil {
return nil, err
}
}
}
return nil, ErrNotExist{"", relpath}
}

104
modules/git/tree_entry.go
View file

@ -9,55 +9,8 @@ import (
"io"
"sort"
"strings"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/object"
)
// EntryMode the type of the object in the git tree
type EntryMode int
// There are only a few file modes in Git. They look like unix file modes, but they can only be
// one of these.
const (
// EntryModeBlob
EntryModeBlob EntryMode = 0100644
// EntryModeExec
EntryModeExec EntryMode = 0100755
// EntryModeSymlink
EntryModeSymlink EntryMode = 0120000
// EntryModeCommit
EntryModeCommit EntryMode = 0160000
// EntryModeTree
EntryModeTree EntryMode = 0040000
)
// TreeEntry the leaf in the git tree
type TreeEntry struct {
ID SHA1
gogitTreeEntry *object.TreeEntry
ptree *Tree
size int64
sized bool
fullName string
}
// Name returns the name of the entry
func (te *TreeEntry) Name() string {
if te.fullName != "" {
return te.fullName
}
return te.gogitTreeEntry.Name
}
// Mode returns the mode of the entry
func (te *TreeEntry) Mode() EntryMode {
return EntryMode(te.gogitTreeEntry.Mode)
}
// Type returns the type of the entry (commit, tree, blob)
func (te *TreeEntry) Type() string {
switch te.Mode() {
@ -70,63 +23,6 @@ func (te *TreeEntry) Type() string {
}
}
// Size returns the size of the entry
func (te *TreeEntry) Size() int64 {
if te.IsDir() {
return 0
} else if te.sized {
return te.size
}
file, err := te.ptree.gogitTree.TreeEntryFile(te.gogitTreeEntry)
if err != nil {
return 0
}
te.sized = true
te.size = file.Size
return te.size
}
// IsSubModule if the entry is a sub module
func (te *TreeEntry) IsSubModule() bool {
return te.gogitTreeEntry.Mode == filemode.Submodule
}
// IsDir if the entry is a sub dir
func (te *TreeEntry) IsDir() bool {
return te.gogitTreeEntry.Mode == filemode.Dir
}
// IsLink if the entry is a symlink
func (te *TreeEntry) IsLink() bool {
return te.gogitTreeEntry.Mode == filemode.Symlink
}
// IsRegular if the entry is a regular file
func (te *TreeEntry) IsRegular() bool {
return te.gogitTreeEntry.Mode == filemode.Regular
}
// IsExecutable if the entry is an executable file (not necessarily binary)
func (te *TreeEntry) IsExecutable() bool {
return te.gogitTreeEntry.Mode == filemode.Executable
}
// Blob returns the blob object the entry
func (te *TreeEntry) Blob() *Blob {
encodedObj, err := te.ptree.repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, te.gogitTreeEntry.Hash)
if err != nil {
return nil
}
return &Blob{
ID: te.gogitTreeEntry.Hash,
gogitEncodedObj: encodedObj,
name: te.Name(),
}
}
// FollowLink returns the entry pointed to by a symlink
func (te *TreeEntry) FollowLink() (*TreeEntry, error) {
if !te.IsLink() {

96
modules/git/tree_entry_gogit.go Normal file
View file

@ -0,0 +1,96 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/object"
)
// TreeEntry the leaf in the git tree
type TreeEntry struct {
ID SHA1
gogitTreeEntry *object.TreeEntry
ptree *Tree
size int64
sized bool
fullName string
}
// Name returns the name of the entry
func (te *TreeEntry) Name() string {
if te.fullName != "" {
return te.fullName
}
return te.gogitTreeEntry.Name
}
// Mode returns the mode of the entry
func (te *TreeEntry) Mode() EntryMode {
return EntryMode(te.gogitTreeEntry.Mode)
}
// Size returns the size of the entry
func (te *TreeEntry) Size() int64 {
if te.IsDir() {
return 0
} else if te.sized {
return te.size
}
file, err := te.ptree.gogitTree.TreeEntryFile(te.gogitTreeEntry)
if err != nil {
return 0
}
te.sized = true
te.size = file.Size
return te.size
}
// IsSubModule if the entry is a sub module
func (te *TreeEntry) IsSubModule() bool {
return te.gogitTreeEntry.Mode == filemode.Submodule
}
// IsDir if the entry is a sub dir
func (te *TreeEntry) IsDir() bool {
return te.gogitTreeEntry.Mode == filemode.Dir
}
// IsLink if the entry is a symlink
func (te *TreeEntry) IsLink() bool {
return te.gogitTreeEntry.Mode == filemode.Symlink
}
// IsRegular if the entry is a regular file
func (te *TreeEntry) IsRegular() bool {
return te.gogitTreeEntry.Mode == filemode.Regular
}
// IsExecutable if the entry is an executable file (not necessarily binary)
func (te *TreeEntry) IsExecutable() bool {
return te.gogitTreeEntry.Mode == filemode.Executable
}
// Blob returns the blob object the entry
func (te *TreeEntry) Blob() *Blob {
encodedObj, err := te.ptree.repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, te.gogitTreeEntry.Hash)
if err != nil {
return nil
}
return &Blob{
ID: te.gogitTreeEntry.Hash,
gogitEncodedObj: encodedObj,
name: te.Name(),
}
}

36
modules/git/tree_entry_mode.go Normal file
View file

@ -0,0 +1,36 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package git
import "strconv"
// EntryMode the type of the object in the git tree
type EntryMode int
// There are only a few file modes in Git. They look like unix file modes, but they can only be
// one of these.
const (
// EntryModeBlob
EntryModeBlob EntryMode = 0100644
// EntryModeExec
EntryModeExec EntryMode = 0100755
// EntryModeSymlink
EntryModeSymlink EntryMode = 0120000
// EntryModeCommit
EntryModeCommit EntryMode = 0160000
// EntryModeTree
EntryModeTree EntryMode = 0040000
)
// String converts an EntryMode to a string
func (e EntryMode) String() string {
return strconv.FormatInt(int64(e), 8)
}
// ToEntryMode converts a string to an EntryMode
func ToEntryMode(value string) EntryMode {
v, _ := strconv.ParseInt(value, 8, 32)
return EntryMode(v)
}

91
modules/git/tree_entry_nogogit.go Normal file
View file

@ -0,0 +1,91 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"strconv"
"strings"
)
// TreeEntry the leaf in the git tree
type TreeEntry struct {
ID SHA1
ptree *Tree
entryMode EntryMode
name string
size int64
sized bool
fullName string
}
// Name returns the name of the entry
func (te *TreeEntry) Name() string {
if te.fullName != "" {
return te.fullName
}
return te.name
}
// Mode returns the mode of the entry
func (te *TreeEntry) Mode() EntryMode {
return te.entryMode
}
// Size returns the size of the entry
func (te *TreeEntry) Size() int64 {
if te.IsDir() {
return 0
} else if te.sized {
return te.size
}
stdout, err := NewCommand("cat-file", "-s", te.ID.String()).RunInDir(te.ptree.repo.Path)
if err != nil {
return 0
}
te.sized = true
te.size, _ = strconv.ParseInt(strings.TrimSpace(stdout), 10, 64)
return te.size
}
// IsSubModule if the entry is a sub module
func (te *TreeEntry) IsSubModule() bool {
return te.entryMode == EntryModeCommit
}
// IsDir if the entry is a sub dir
func (te *TreeEntry) IsDir() bool {
return te.entryMode == EntryModeTree
}
// IsLink if the entry is a symlink
func (te *TreeEntry) IsLink() bool {
return te.entryMode == EntryModeSymlink
}
// IsRegular if the entry is a regular file
func (te *TreeEntry) IsRegular() bool {
return te.entryMode == EntryModeBlob
}
// IsExecutable if the entry is an executable file (not necessarily binary)
func (te *TreeEntry) IsExecutable() bool {
return te.entryMode == EntryModeExec
}
// Blob returns the blob object the entry
func (te *TreeEntry) Blob() *Blob {
return &Blob{
ID: te.ID,
repoPath: te.ptree.repo.Path,
name: te.Name(),
}
}

2
modules/git/tree_entry_test.go
View file

@ -2,6 +2,8 @@
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (

94
modules/git/tree_gogit.go Normal file
View file

@ -0,0 +1,94 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"io"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
)
// Tree represents a flat directory listing.
type Tree struct {
ID SHA1
ResolvedID SHA1
repo *Repository
gogitTree *object.Tree
// parent tree
ptree *Tree
}
func (t *Tree) loadTreeObject() error {
gogitTree, err := t.repo.gogitRepo.TreeObject(t.ID)
if err != nil {
return err
}
t.gogitTree = gogitTree
return nil
}
// ListEntries returns all entries of current tree.
func (t *Tree) ListEntries() (Entries, error) {
if t.gogitTree == nil {
err := t.loadTreeObject()
if err != nil {
return nil, err
}
}
entries := make([]*TreeEntry, len(t.gogitTree.Entries))
for i, entry := range t.gogitTree.Entries {
entries[i] = &TreeEntry{
ID: entry.Hash,
gogitTreeEntry: &t.gogitTree.Entries[i],
ptree: t,
}
}
return entries, nil
}
// ListEntriesRecursive returns all entries of current tree recursively including all subtrees
func (t *Tree) ListEntriesRecursive() (Entries, error) {
if t.gogitTree == nil {
err := t.loadTreeObject()
if err != nil {
return nil, err
}
}
var entries []*TreeEntry
seen := map[plumbing.Hash]bool{}
walker := object.NewTreeWalker(t.gogitTree, true, seen)
for {
fullName, entry, err := walker.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
if seen[entry.Hash] {
continue
}
convertedEntry := &TreeEntry{
ID: entry.Hash,
gogitTreeEntry: &entry,
ptree: t,
fullName: fullName,
}
entries = append(entries, convertedEntry)
}
return entries, nil
}

69
modules/git/tree_nogogit.go Normal file
View file

@ -0,0 +1,69 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"strings"
)
// Tree represents a flat directory listing.
type Tree struct {
ID SHA1
ResolvedID SHA1
repo *Repository
// parent tree
ptree *Tree
entries Entries
entriesParsed bool
entriesRecursive Entries
entriesRecursiveParsed bool
}
// ListEntries returns all entries of current tree.
func (t *Tree) ListEntries() (Entries, error) {
if t.entriesParsed {
return t.entries, nil
}
stdout, err := NewCommand("ls-tree", t.ID.String()).RunInDirBytes(t.repo.Path)
if err != nil {
if strings.Contains(err.Error(), "fatal: Not a valid object name") || strings.Contains(err.Error(), "fatal: not a tree object") {
return nil, ErrNotExist{
ID: t.ID.String(),
}
}
return nil, err
}
t.entries, err = parseTreeEntries(stdout, t)
if err == nil {
t.entriesParsed = true
}
return t.entries, err
}
// ListEntriesRecursive returns all entries of current tree recursively including all subtrees
func (t *Tree) ListEntriesRecursive() (Entries, error) {
if t.entriesRecursiveParsed {
return t.entriesRecursive, nil
}
stdout, err := NewCommand("ls-tree", "-t", "-r", t.ID.String()).RunInDirBytes(t.repo.Path)
if err != nil {
return nil, err
}
t.entriesRecursive, err = parseTreeEntries(stdout, t)
if err == nil {
t.entriesRecursiveParsed = true
}
return t.entriesRecursive, err
}

32
modules/git/utils.go
View file

@ -6,6 +6,7 @@ package git
import (
"fmt"
"io"
"os"
"strconv"
"strings"
@ -68,11 +69,12 @@ func isExist(path string) bool {
return err == nil || os.IsExist(err)
}
func concatenateError(err error, stderr string) error {
// ConcatenateError concatenats an error with stderr string
func ConcatenateError(err error, stderr string) error {
if len(stderr) == 0 {
return err
}
return fmt.Errorf("%v - %s", err, stderr)
return fmt.Errorf("%w - %s", err, stderr)
}
// RefEndName return the end name of a ref name
@ -140,3 +142,29 @@ func ParseBool(value string) (result bool, valid bool) {
}
return intValue != 0, true
}
// LimitedReaderCloser is a limited reader closer
type LimitedReaderCloser struct {
R io.Reader
C io.Closer
N int64
}
// Read implements io.Reader
func (l *LimitedReaderCloser) Read(p []byte) (n int, err error) {
if l.N <= 0 {
_ = l.C.Close()
return 0, io.EOF
}
if int64(len(p)) > l.N {
p = p[0:l.N]
}
n, err = l.R.Read(p)
l.N -= int64(n)
return
}
// Close implements io.Closer
func (l *LimitedReaderCloser) Close() error {
return l.C.Close()
}

3
modules/indexer/stats/db.go
View file

@ -7,6 +7,7 @@ package stats
import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
)
// DBIndexer implements Indexer interface to use database's like search
@ -37,6 +38,7 @@ func (db *DBIndexer) Index(id int64) error {
// Get latest commit for default branch
commitID, err := gitRepo.GetBranchCommitID(repo.DefaultBranch)
if err != nil {
log.Error("Unable to get commit ID for defaultbranch %s in %s", repo.DefaultBranch, repo.RepoPath())
return err
}
@ -48,6 +50,7 @@ func (db *DBIndexer) Index(id int64) error {
// Calculate and save language statistics to database
stats, err := gitRepo.GetLanguageStats(commitID)
if err != nil {
log.Error("Unable to get language stats for ID %s for defaultbranch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err)
return err
}
return repo.UpdateLanguageStats(commitID, stats)

54
modules/repository/cache.go
View file

@ -5,57 +5,14 @@
package repository
import (
"path"
"strings"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/setting"
cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph"
)
func recusiveCache(gitRepo *git.Repository, c cgobject.CommitNode, tree *git.Tree, treePath string, ca *cache.LastCommitCache, level int) error {
if level == 0 {
return nil
}
entries, err := tree.ListEntries()
if err != nil {
return err
}
entryPaths := make([]string, len(entries))
entryMap := make(map[string]*git.TreeEntry)
for i, entry := range entries {
entryPaths[i] = entry.Name()
entryMap[entry.Name()] = entry
}
commits, err := git.GetLastCommitForPaths(c, treePath, entryPaths)
if err != nil {
return err
}
for entry, cm := range commits {
if err := ca.Put(c.ID().String(), path.Join(treePath, entry), cm.ID().String()); err != nil {
return err
}
if entryMap[entry].IsDir() {
subTree, err := tree.SubTree(entry)
if err != nil {
return err
}
if err := recusiveCache(gitRepo, c, subTree, entry, ca, level-1); err != nil {
return err
}
}
}
return nil
}
func getRefName(fullRefName string) string {
if strings.HasPrefix(fullRefName, git.TagPrefix) {
return fullRefName[len(git.TagPrefix):]
@ -84,14 +41,7 @@ func CacheRef(repo *models.Repository, gitRepo *git.Repository, fullRefName stri
return nil
}
commitNodeIndex, _ := gitRepo.CommitNodeIndex()
commitCache := git.NewLastCommitCache(repo.FullName(), gitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()), cache.GetCache())
c, err := commitNodeIndex.Get(commit.ID)
if err != nil {
return err
}
ca := cache.NewLastCommitCache(repo.FullName(), gitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()))
return recusiveCache(gitRepo, c, &commit.Tree, "", ca, 1)
return commitCache.CacheCommit(commit)
}

3
routers/private/hook.go
View file

@ -25,7 +25,6 @@ import (
repo_service "code.gitea.io/gitea/services/repository"
"gitea.com/macaron/macaron"
"github.com/go-git/go-git/v5/plumbing"
)
func verifyCommits(oldCommitID, newCommitID string, repo *git.Repository, env []string) error {
@ -82,7 +81,7 @@ func readAndVerifyCommit(sha string, repo *git.Repository, env []string) error {
_ = stdoutReader.Close()
_ = stdoutWriter.Close()
}()
hash := plumbing.NewHash(sha)
hash := git.MustIDFromString(sha)
return git.NewCommand("cat-file", "commit", sha).
RunInDirTimeoutEnvFullPipelineFunc(env, -1, repo.Path,

146
routers/repo/lfs.go
View file

@ -12,11 +12,9 @@ import (
"io"
"io/ioutil"
"path"
"sort"
"strconv"
"strings"
"sync"
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
@ -29,9 +27,6 @@ import (
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
gogit "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/unknwon/com"
)
@ -363,22 +358,6 @@ func LFSDelete(ctx *context.Context) {
ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs")
}
type lfsResult struct {
Name string
SHA string
Summary string
When time.Time
ParentHashes []plumbing.Hash
BranchName string
FullCommitName string
}
type lfsResultSlice []*lfsResult
func (a lfsResultSlice) Len() int { return len(a) }
func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) }
// LFSFileFind guesses a sha for the provided oid (or uses the provided sha) and then finds the commits that contain this sha
func LFSFileFind(ctx *context.Context) {
if !setting.LFS.StartServer {
@ -394,140 +373,27 @@ func LFSFileFind(ctx *context.Context) {
sha := ctx.Query("sha")
ctx.Data["Title"] = oid
ctx.Data["PageIsSettingsLFS"] = true
var hash plumbing.Hash
var hash git.SHA1
if len(sha) == 0 {
meta := models.LFSMetaObject{Oid: oid, Size: size}
pointer := meta.Pointer()
hash = plumbing.ComputeHash(plumbing.BlobObject, []byte(pointer))
hash = git.ComputeBlobHash([]byte(pointer))
sha = hash.String()
} else {
hash = plumbing.NewHash(sha)
hash = git.MustIDFromString(sha)
}
ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
ctx.Data["Oid"] = oid
ctx.Data["Size"] = size
ctx.Data["SHA"] = sha
resultsMap := map[string]*lfsResult{}
results := make([]*lfsResult, 0)
basePath := ctx.Repo.Repository.RepoPath()
gogitRepo := ctx.Repo.GitRepo.GoGitRepo()
commitsIter, err := gogitRepo.Log(&gogit.LogOptions{
Order: gogit.LogOrderCommitterTime,
All: true,
})
if err != nil {
log.Error("Failed to get GoGit CommitsIter: %v", err)
ctx.ServerError("LFSFind: Iterate Commits", err)
return
}
err = commitsIter.ForEach(func(gitCommit *object.Commit) error {
tree, err := gitCommit.Tree()
if err != nil {
return err
}
treeWalker := object.NewTreeWalker(tree, true, nil)
defer treeWalker.Close()
for {
name, entry, err := treeWalker.Next()
if err == io.EOF {
break
}
if entry.Hash == hash {
result := lfsResult{
Name: name,
SHA: gitCommit.Hash.String(),
Summary: strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0],
When: gitCommit.Author.When,
ParentHashes: gitCommit.ParentHashes,
}
resultsMap[gitCommit.Hash.String()+":"+name] = &result
}
}
return nil
})
results, err := pipeline.FindLFSFile(ctx.Repo.GitRepo, hash)
if err != nil && err != io.EOF {
log.Error("Failure in CommitIter.ForEach: %v", err)
ctx.ServerError("LFSFind: IterateCommits ForEach", err)
log.Error("Failure in FindLFSFile: %v", err)
ctx.ServerError("LFSFind: FindLFSFile.", err)
return
}
for _, result := range resultsMap {
hasParent := false
for _, parentHash := range result.ParentHashes {
if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent {
break
}
}
if !hasParent {
results = append(results, result)
}
}
sort.Sort(lfsResultSlice(results))
// Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
shasToNameReader, shasToNameWriter := io.Pipe()
nameRevStdinReader, nameRevStdinWriter := io.Pipe()
errChan := make(chan error, 1)
wg := sync.WaitGroup{}
wg.Add(3)
go func() {
defer wg.Done()
scanner := bufio.NewScanner(nameRevStdinReader)
i := 0
for scanner.Scan() {
line := scanner.Text()
if len(line) == 0 {
continue
}
result := results[i]
result.FullCommitName = line
result.BranchName = strings.Split(line, "~")[0]
i++
}
}()
go pipeline.NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath)
go func() {
defer wg.Done()
defer shasToNameWriter.Close()
for _, result := range results {
i := 0
if i < len(result.SHA) {
n, err := shasToNameWriter.Write([]byte(result.SHA)[i:])
if err != nil {
errChan <- err
break
}
i += n
}
n := 0
for n < 1 {
n, err = shasToNameWriter.Write([]byte{'\n'})
if err != nil {
errChan <- err
break
}
}
}
}()
wg.Wait()
select {
case err, has := <-errChan:
if has {
ctx.ServerError("LFSPointerFiles", err)
}
default:
}
ctx.Data["Results"] = results
ctx.HTML(200, tplSettingsLFSFileFind)
}

4
routers/repo/view.go
View file

@ -137,9 +137,9 @@ func renderDirectory(ctx *context.Context, treeLink string) {
}
entries.CustomSort(base.NaturalSortLess)
var c git.LastCommitCache
var c *git.LastCommitCache
if setting.CacheService.LastCommit.Enabled && ctx.Repo.CommitsCount >= setting.CacheService.LastCommit.CommitsCount {
c = cache.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()))
c = git.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()), cache.GetCache())
}
var latestCommit *git.Commit

11
templates/repo/view_list.tmpl
View file

@ -40,18 +40,19 @@
</tr>
{{end}}
{{range $item := .Files}}
{{$entry := index $item 0}}
{{$commit := index $item 1}}
{{$entry := $item.Entry}}
{{$commit := $item.Commit}}
{{$subModuleFile := $item.SubModuleFile}}
<tr>
<td class="name four wide">
<span class="truncate">
{{if $entry.IsSubModule}}
{{svg "octicon-file-submodule"}}
{{$refURL := $commit.RefURL AppUrl $.Repository.FullName $.SSHDomain}}
{{$refURL := $subModuleFile.RefURL AppUrl $.Repository.FullName $.SSHDomain}}
{{if $refURL}}
<a href="{{$refURL}}">{{$entry.Name}}</a><span class="at">@</span><a href="{{$refURL}}/commit/{{$commit.RefID}}">{{ShortSha $commit.RefID}}</a>
<a href="{{$refURL}}">{{$entry.Name}}</a><span class="at">@</span><a href="{{$refURL}}/commit/{{$subModuleFile.RefID}}">{{ShortSha $subModuleFile.RefID}}</a>
{{else}}
{{$entry.Name}}<span class="at">@</span>{{ShortSha $commit.RefID}}
{{$entry.Name}}<span class="at">@</span>{{ShortSha $subModuleFile.RefID}}
{{end}}
{{else}}
{{if $entry.IsDir}}