From 37a300b105a0178f28693abcb5466e808f3f9627 Mon Sep 17 00:00:00 2001 From: dragon Date: Fri, 6 Dec 2024 14:51:49 +0800 Subject: [PATCH] Add support for indexing arch files --- modules/packages/arch/metadata.go | 25 ++++++++++- modules/packages/arch/metadata_test.go | 14 ++++-- routers/api/packages/arch/arch.go | 3 +- services/packages/arch/repository.go | 36 +++++++++++++--- tests/integration/api_packages_arch_test.go | 47 ++++++++++++++------- 5 files changed, 96 insertions(+), 29 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 6cdde75cdc..6a044002a8 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -4,6 +4,7 @@ package arch import ( + "archive/tar" "bufio" "bytes" "encoding/hex" @@ -25,7 +26,9 @@ import ( // https://man.archlinux.org/man/PKGBUILD.5 const ( - PropertyDescription = "arch.description" + PropertyDescription = "arch.description" + PropertyFiles = "arch.files" + PropertyArch = "arch.architecture" PropertyDistribution = "arch.distribution" @@ -85,6 +88,8 @@ type FileMetadata struct { Packager string `json:"packager"` Arch string `json:"arch"` PgpSigned string `json:"pgp"` + + Files []string `json:"files,omitempty"` } // ParsePackage Function that receives arch package archive data and returns it's metadata. @@ -127,6 +132,8 @@ func ParsePackage(r *packages.HashedBuffer) (*Package, error) { var pkg *Package var mTree bool + files := make([]string, 0) + for { f, err := tarball.Read() if err == io.EOF { @@ -135,6 +142,11 @@ func ParsePackage(r *packages.HashedBuffer) (*Package, error) { if err != nil { return nil, err } + // ref:https://gitlab.archlinux.org/pacman/pacman/-/blob/91546004903eea5d5267d59898a6029ba1d64031/lib/libalpm/add.c#L529-L533 + if !strings.HasPrefix(f.Name(), ".") { + files = append(files, (f.Header.(*tar.Header)).Name) + } + switch f.Name() { case ".PKGINFO": pkg, err = ParsePackageInfo(tarballType, f) @@ -155,7 +167,7 @@ func ParsePackage(r *packages.HashedBuffer) (*Package, error) { if !mTree { return nil, util.NewInvalidArgumentErrorf(".MTREE file not found") } - + pkg.FileMetadata.Files = files pkg.FileMetadata.CompressedSize = r.Size() pkg.FileMetadata.MD5 = hex.EncodeToString(md5) pkg.FileMetadata.SHA256 = hex.EncodeToString(sha256) @@ -339,3 +351,12 @@ func (p *Package) Desc() string { } return buf.String() } + +func (p *Package) Files() string { + var buf bytes.Buffer + buf.WriteString("%FILES%\n") + for _, item := range p.FileMetadata.Files { + _, _ = fmt.Fprintf(&buf, "%s\n", item) + } + return buf.String() +} diff --git a/modules/packages/arch/metadata_test.go b/modules/packages/arch/metadata_test.go index ddb35ca837..eb8de333ad 100644 --- a/modules/packages/arch/metadata_test.go +++ b/modules/packages/arch/metadata_test.go @@ -344,8 +344,8 @@ func TestValidatePackageSpec(t *testing.T) { }) } -func TestDescString(t *testing.T) { - const pkgdesc = `%FILENAME% +func TestDescAndFileString(t *testing.T) { + const pkgDesc = `%FILENAME% zstd-1.5.5-1-x86_64.pkg.tar.zst %NAME% @@ -415,6 +415,12 @@ ninja dummy5 dummy6 +` + + const pkgFiles = `%FILES% +usr/ +usr/bin/ +usr/bin/zstd ` md := &Package{ @@ -441,7 +447,9 @@ dummy6 BuildDate: 1681646714, Packager: "Jelle van der Waa ", Arch: "x86_64", + Files: []string{"usr/", "usr/bin/", "usr/bin/zstd"}, }, } - require.Equal(t, pkgdesc, md.Desc()) + require.Equal(t, pkgDesc, md.Desc()) + require.Equal(t, pkgFiles, md.Files()) } diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index ecd2281884..1e8595f5c0 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -26,7 +26,7 @@ import ( var ( archPkgOrSig = regexp.MustCompile(`^.*\.pkg\.tar\.\w+(\.sig)*$`) - archDBOrSig = regexp.MustCompile(`^.*.db(\.tar\.gz)*(\.sig)*$`) + archDBOrSig = regexp.MustCompile(`^.*.(db|files)(\.tar\.gz)*(\.sig)*$`) locker = sync.NewExclusivePool() ) @@ -115,6 +115,7 @@ func PushPackage(ctx *context.Context) { properties := map[string]string{ arch_module.PropertyDescription: p.Desc(), + arch_module.PropertyFiles: p.Files(), arch_module.PropertyArch: p.FileMetadata.Arch, arch_module.PropertyDistribution: group, } diff --git a/services/packages/arch/repository.go b/services/packages/arch/repository.go index 763a0a2102..e681f24561 100644 --- a/services/packages/arch/repository.go +++ b/services/packages/arch/repository.go @@ -225,22 +225,44 @@ func createDB(ctx context.Context, ownerID int64, group, arch string) (*packages if err != nil { return nil, err } - if len(pps) >= 1 { - meta := []byte(pps[0].Value) + if len(pps) == 0 { + continue + } + pkgDesc := []byte(pps[0].Value) + header := &tar.Header{ + Name: pkg.Name + "-" + ver.Version + "/desc", + Size: int64(len(pkgDesc)), + Mode: int64(os.ModePerm), + } + if err = tw.WriteHeader(header); err != nil { + return nil, err + } + if _, err := tw.Write(pkgDesc); err != nil { + return nil, err + } + + pfs, err := packages_model.GetPropertiesByName( + ctx, packages_model.PropertyTypeFile, pf.ID, arch_module.PropertyFiles, + ) + if err != nil { + return nil, err + } + if len(pfs) >= 1 { + pkgFiles := []byte(pfs[0].Value) header := &tar.Header{ - Name: pkg.Name + "-" + ver.Version + "/desc", - Size: int64(len(meta)), + Name: pkg.Name + "-" + ver.Version + "/files", + Size: int64(len(pkgFiles)), Mode: int64(os.ModePerm), } if err = tw.WriteHeader(header); err != nil { return nil, err } - if _, err := tw.Write(meta); err != nil { + if _, err := tw.Write(pkgFiles); err != nil { return nil, err } - count++ - break } + count++ + break } } if count == 0 { diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 8651af4a58..57db324a2f 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -223,8 +223,14 @@ HMhNSS1IzUsBcpJAPFAwwUXSM0u4BjoaR8EoGAWjgGQAAILFeyQADAAA t.Run(fmt.Sprintf("RepositoryDB[%s]", group), func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", rootURL+"/repository.key") - respPub := MakeRequest(t, req, http.StatusOK) + req := NewRequest(t, "GET", groupURL+"/x86_64/base.db.tar.gz") + MakeRequest(t, req, http.StatusOK) + + req = NewRequest(t, "GET", groupURL+"/x86_64/base.files") + MakeRequest(t, req, http.StatusOK) + + req = NewRequest(t, "GET", groupURL+"/x86_64/base.files.tar.gz") + MakeRequest(t, req, http.StatusOK) req = NewRequest(t, "GET", groupURL+"/x86_64/base.db") respPkg := MakeRequest(t, req, http.StatusOK) @@ -232,23 +238,32 @@ HMhNSS1IzUsBcpJAPFAwwUXSM0u4BjoaR8EoGAWjgGQAAILFeyQADAAA req = NewRequest(t, "GET", groupURL+"/x86_64/base.db.sig") respSig := MakeRequest(t, req, http.StatusOK) + req = NewRequest(t, "GET", rootURL+"/repository.key") + respPub := MakeRequest(t, req, http.StatusOK) + if err := gpgVerify(respPub.Body.Bytes(), respSig.Body.Bytes(), respPkg.Body.Bytes()); err != nil { t.Fatal(err) } files, err := listTarGzFiles(respPkg.Body.Bytes()) require.NoError(t, err) - require.Len(t, files, 1) + require.Len(t, files, 2) for s, d := range files { - name := getProperty(string(d.Data), "NAME") - ver := getProperty(string(d.Data), "VERSION") - require.Equal(t, name+"-"+ver+"/desc", s) - fn := getProperty(string(d.Data), "FILENAME") - pgp := getProperty(string(d.Data), "PGPSIG") - req = NewRequest(t, "GET", groupURL+"/x86_64/"+fn+".sig") - respSig := MakeRequest(t, req, http.StatusOK) - decodeString, err := base64.StdEncoding.DecodeString(pgp) - require.NoError(t, err) - require.Equal(t, respSig.Body.Bytes(), decodeString) + if strings.HasSuffix(s, "/desc") { + name := getProperty(string(d.Data), "NAME") + ver := getProperty(string(d.Data), "VERSION") + require.Equal(t, name+"-"+ver+"/desc", s) + fn := getProperty(string(d.Data), "FILENAME") + pgp := getProperty(string(d.Data), "PGPSIG") + req = NewRequest(t, "GET", groupURL+"/x86_64/"+fn+".sig") + respSig := MakeRequest(t, req, http.StatusOK) + decodeString, err := base64.StdEncoding.DecodeString(pgp) + require.NoError(t, err) + require.Equal(t, respSig.Body.Bytes(), decodeString) + } else if strings.HasSuffix(s, "/files") { + require.True(t, strings.HasPrefix(string(d.Data), "%FILES%")) + } else { + require.Failf(t, "unknown item", "fileName:%s", s) + } } }) @@ -275,7 +290,7 @@ HMhNSS1IzUsBcpJAPFAwwUXSM0u4BjoaR8EoGAWjgGQAAILFeyQADAAA respPkg := MakeRequest(t, req, http.StatusOK) files, err := listTarGzFiles(respPkg.Body.Bytes()) require.NoError(t, err) - require.Len(t, files, 1) + require.Len(t, files, 2) req = NewRequestWithBody(t, "DELETE", groupURL+"/test2/1.0.0-1/any", nil). AddBasicAuth(user.Name) @@ -347,7 +362,7 @@ HMhNSS1IzUsBcpJAPFAwwUXSM0u4BjoaR8EoGAWjgGQAAILFeyQADAAA files, err := listTarGzFiles(respPkg.Body.Bytes()) require.NoError(t, err) - require.Len(t, files, 1) + require.Len(t, files, 2) req = NewRequestWithBody(t, "PUT", rootURL, bytes.NewReader(pkgs["otherXZ"])). AddBasicAuth(user.Name) @@ -358,7 +373,7 @@ HMhNSS1IzUsBcpJAPFAwwUXSM0u4BjoaR8EoGAWjgGQAAILFeyQADAAA files, err = listTarGzFiles(respPkg.Body.Bytes()) require.NoError(t, err) - require.Len(t, files, 2) + require.Len(t, files, 4) }) }