mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2024-11-27 09:11:53 -05:00
460a2b0edf
Currently, Artifact does not have an expiration and automatic cleanup mechanism, and this feature needs to be added. It contains the following key points: - [x] add global artifact retention days option in config file. Default value is 90 days. - [x] add cron task to clean up expired artifacts. It should run once a day. - [x] support custom retention period from `retention-days: 5` in `upload-artifact@v3`. - [x] artifacts link in actions view should be non-clickable text when expired.
455 lines
14 KiB
Go
455 lines
14 KiB
Go
// Copyright 2023 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package actions
|
|
|
|
// GitHub Actions Artifacts API Simple Description
|
|
//
|
|
// 1. Upload artifact
|
|
// 1.1. Post upload url
|
|
// Post: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts?api-version=6.0-preview
|
|
// Request:
|
|
// {
|
|
// "Type": "actions_storage",
|
|
// "Name": "artifact"
|
|
// }
|
|
// Response:
|
|
// {
|
|
// "fileContainerResourceUrl":"/api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/upload"
|
|
// }
|
|
// it acquires an upload url for artifact upload
|
|
// 1.2. Upload artifact
|
|
// PUT: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/upload?itemPath=artifact%2Ffilename
|
|
// it upload chunk with headers:
|
|
// x-tfs-filelength: 1024 // total file length
|
|
// content-length: 1024 // chunk length
|
|
// x-actions-results-md5: md5sum // md5sum of chunk
|
|
// content-range: bytes 0-1023/1024 // chunk range
|
|
// we save all chunks to one storage directory after md5sum check
|
|
// 1.3. Confirm upload
|
|
// PATCH: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/upload?itemPath=artifact%2Ffilename
|
|
// it confirm upload and merge all chunks to one file, save this file to storage
|
|
//
|
|
// 2. Download artifact
|
|
// 2.1 list artifacts
|
|
// GET: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts?api-version=6.0-preview
|
|
// Response:
|
|
// {
|
|
// "count": 1,
|
|
// "value": [
|
|
// {
|
|
// "name": "artifact",
|
|
// "fileContainerResourceUrl": "/api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/path"
|
|
// }
|
|
// ]
|
|
// }
|
|
// 2.2 download artifact
|
|
// GET: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/path?api-version=6.0-preview
|
|
// Response:
|
|
// {
|
|
// "value": [
|
|
// {
|
|
// "contentLocation": "/api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/download",
|
|
// "path": "artifact/filename",
|
|
// "itemType": "file"
|
|
// }
|
|
// ]
|
|
// }
|
|
// 2.3 download artifact file
|
|
// GET: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/download?itemPath=artifact%2Ffilename
|
|
// Response:
|
|
// download file
|
|
//
|
|
|
|
import (
|
|
"crypto/md5"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"code.gitea.io/gitea/models/actions"
|
|
"code.gitea.io/gitea/modules/context"
|
|
"code.gitea.io/gitea/modules/json"
|
|
"code.gitea.io/gitea/modules/log"
|
|
"code.gitea.io/gitea/modules/setting"
|
|
"code.gitea.io/gitea/modules/storage"
|
|
"code.gitea.io/gitea/modules/util"
|
|
"code.gitea.io/gitea/modules/web"
|
|
web_types "code.gitea.io/gitea/modules/web/types"
|
|
)
|
|
|
|
const artifactRouteBase = "/_apis/pipelines/workflows/{run_id}/artifacts"
|
|
|
|
type artifactContextKeyType struct{}
|
|
|
|
var artifactContextKey = artifactContextKeyType{}
|
|
|
|
type ArtifactContext struct {
|
|
*context.Base
|
|
|
|
ActionTask *actions.ActionTask
|
|
}
|
|
|
|
func init() {
|
|
web.RegisterResponseStatusProvider[*ArtifactContext](func(req *http.Request) web_types.ResponseStatusProvider {
|
|
return req.Context().Value(artifactContextKey).(*ArtifactContext)
|
|
})
|
|
}
|
|
|
|
func ArtifactsRoutes(prefix string) *web.Route {
|
|
m := web.NewRoute()
|
|
m.Use(ArtifactContexter())
|
|
|
|
r := artifactRoutes{
|
|
prefix: prefix,
|
|
fs: storage.ActionsArtifacts,
|
|
}
|
|
|
|
m.Group(artifactRouteBase, func() {
|
|
// retrieve, list and confirm artifacts
|
|
m.Combo("").Get(r.listArtifacts).Post(r.getUploadArtifactURL).Patch(r.comfirmUploadArtifact)
|
|
// handle container artifacts list and download
|
|
m.Put("/{artifact_hash}/upload", r.uploadArtifact)
|
|
// handle artifacts download
|
|
m.Get("/{artifact_hash}/download_url", r.getDownloadArtifactURL)
|
|
m.Get("/{artifact_id}/download", r.downloadArtifact)
|
|
})
|
|
|
|
return m
|
|
}
|
|
|
|
func ArtifactContexter() func(next http.Handler) http.Handler {
|
|
return func(next http.Handler) http.Handler {
|
|
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
|
|
base, baseCleanUp := context.NewBaseContext(resp, req)
|
|
defer baseCleanUp()
|
|
|
|
ctx := &ArtifactContext{Base: base}
|
|
ctx.AppendContextValue(artifactContextKey, ctx)
|
|
|
|
// action task call server api with Bearer ACTIONS_RUNTIME_TOKEN
|
|
// we should verify the ACTIONS_RUNTIME_TOKEN
|
|
authHeader := req.Header.Get("Authorization")
|
|
if len(authHeader) == 0 || !strings.HasPrefix(authHeader, "Bearer ") {
|
|
ctx.Error(http.StatusUnauthorized, "Bad authorization header")
|
|
return
|
|
}
|
|
|
|
authToken := strings.TrimPrefix(authHeader, "Bearer ")
|
|
task, err := actions.GetRunningTaskByToken(req.Context(), authToken)
|
|
if err != nil {
|
|
log.Error("Error runner api getting task: %v", err)
|
|
ctx.Error(http.StatusInternalServerError, "Error runner api getting task")
|
|
return
|
|
}
|
|
|
|
if err := task.LoadJob(req.Context()); err != nil {
|
|
log.Error("Error runner api getting job: %v", err)
|
|
ctx.Error(http.StatusInternalServerError, "Error runner api getting job")
|
|
return
|
|
}
|
|
|
|
ctx.ActionTask = task
|
|
next.ServeHTTP(ctx.Resp, ctx.Req)
|
|
})
|
|
}
|
|
}
|
|
|
|
type artifactRoutes struct {
|
|
prefix string
|
|
fs storage.ObjectStorage
|
|
}
|
|
|
|
func (ar artifactRoutes) buildArtifactURL(runID int64, artifactHash, suffix string) string {
|
|
uploadURL := strings.TrimSuffix(setting.AppURL, "/") + strings.TrimSuffix(ar.prefix, "/") +
|
|
strings.ReplaceAll(artifactRouteBase, "{run_id}", strconv.FormatInt(runID, 10)) +
|
|
"/" + artifactHash + "/" + suffix
|
|
return uploadURL
|
|
}
|
|
|
|
type getUploadArtifactRequest struct {
|
|
Type string
|
|
Name string
|
|
RetentionDays int64
|
|
}
|
|
|
|
type getUploadArtifactResponse struct {
|
|
FileContainerResourceURL string `json:"fileContainerResourceUrl"`
|
|
}
|
|
|
|
// getUploadArtifactURL generates a URL for uploading an artifact
|
|
func (ar artifactRoutes) getUploadArtifactURL(ctx *ArtifactContext) {
|
|
_, runID, ok := validateRunID(ctx)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
var req getUploadArtifactRequest
|
|
if err := json.NewDecoder(ctx.Req.Body).Decode(&req); err != nil {
|
|
log.Error("Error decode request body: %v", err)
|
|
ctx.Error(http.StatusInternalServerError, "Error decode request body")
|
|
return
|
|
}
|
|
|
|
// set retention days
|
|
retentionQuery := ""
|
|
if req.RetentionDays > 0 {
|
|
retentionQuery = fmt.Sprintf("?retentionDays=%d", req.RetentionDays)
|
|
}
|
|
|
|
// use md5(artifact_name) to create upload url
|
|
artifactHash := fmt.Sprintf("%x", md5.Sum([]byte(req.Name)))
|
|
resp := getUploadArtifactResponse{
|
|
FileContainerResourceURL: ar.buildArtifactURL(runID, artifactHash, "upload"+retentionQuery),
|
|
}
|
|
log.Debug("[artifact] get upload url: %s", resp.FileContainerResourceURL)
|
|
ctx.JSON(http.StatusOK, resp)
|
|
}
|
|
|
|
func (ar artifactRoutes) uploadArtifact(ctx *ArtifactContext) {
|
|
task, runID, ok := validateRunID(ctx)
|
|
if !ok {
|
|
return
|
|
}
|
|
artifactName, artifactPath, ok := parseArtifactItemPath(ctx)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
// get upload file size
|
|
fileRealTotalSize, contentLength, err := getUploadFileSize(ctx)
|
|
if err != nil {
|
|
log.Error("Error get upload file size: %v", err)
|
|
ctx.Error(http.StatusInternalServerError, "Error get upload file size")
|
|
return
|
|
}
|
|
|
|
// get artifact retention days
|
|
expiredDays := setting.Actions.ArtifactRetentionDays
|
|
if queryRetentionDays := ctx.Req.URL.Query().Get("retentionDays"); queryRetentionDays != "" {
|
|
expiredDays, err = strconv.ParseInt(queryRetentionDays, 10, 64)
|
|
if err != nil {
|
|
log.Error("Error parse retention days: %v", err)
|
|
ctx.Error(http.StatusBadRequest, "Error parse retention days")
|
|
return
|
|
}
|
|
}
|
|
log.Debug("[artifact] upload chunk, name: %s, path: %s, size: %d, retention days: %d",
|
|
artifactName, artifactPath, fileRealTotalSize, expiredDays)
|
|
|
|
// create or get artifact with name and path
|
|
artifact, err := actions.CreateArtifact(ctx, task, artifactName, artifactPath, expiredDays)
|
|
if err != nil {
|
|
log.Error("Error create or get artifact: %v", err)
|
|
ctx.Error(http.StatusInternalServerError, "Error create or get artifact")
|
|
return
|
|
}
|
|
|
|
// save chunk to storage, if success, return chunk stotal size
|
|
// if artifact is not gzip when uploading, chunksTotalSize == fileRealTotalSize
|
|
// if artifact is gzip when uploading, chunksTotalSize < fileRealTotalSize
|
|
chunksTotalSize, err := saveUploadChunk(ar.fs, ctx, artifact, contentLength, runID)
|
|
if err != nil {
|
|
log.Error("Error save upload chunk: %v", err)
|
|
ctx.Error(http.StatusInternalServerError, "Error save upload chunk")
|
|
return
|
|
}
|
|
|
|
// update artifact size if zero
|
|
if artifact.FileSize == 0 || artifact.FileCompressedSize == 0 {
|
|
artifact.FileSize = fileRealTotalSize
|
|
artifact.FileCompressedSize = chunksTotalSize
|
|
artifact.ContentEncoding = ctx.Req.Header.Get("Content-Encoding")
|
|
if err := actions.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil {
|
|
log.Error("Error update artifact: %v", err)
|
|
ctx.Error(http.StatusInternalServerError, "Error update artifact")
|
|
return
|
|
}
|
|
}
|
|
|
|
ctx.JSON(http.StatusOK, map[string]string{
|
|
"message": "success",
|
|
})
|
|
}
|
|
|
|
// comfirmUploadArtifact comfirm upload artifact.
|
|
// if all chunks are uploaded, merge them to one file.
|
|
func (ar artifactRoutes) comfirmUploadArtifact(ctx *ArtifactContext) {
|
|
_, runID, ok := validateRunID(ctx)
|
|
if !ok {
|
|
return
|
|
}
|
|
artifactName := ctx.Req.URL.Query().Get("artifactName")
|
|
if artifactName == "" {
|
|
log.Error("Error artifact name is empty")
|
|
ctx.Error(http.StatusBadRequest, "Error artifact name is empty")
|
|
return
|
|
}
|
|
if err := mergeChunksForRun(ctx, ar.fs, runID, artifactName); err != nil {
|
|
log.Error("Error merge chunks: %v", err)
|
|
ctx.Error(http.StatusInternalServerError, "Error merge chunks")
|
|
return
|
|
}
|
|
ctx.JSON(http.StatusOK, map[string]string{
|
|
"message": "success",
|
|
})
|
|
}
|
|
|
|
type (
|
|
listArtifactsResponse struct {
|
|
Count int64 `json:"count"`
|
|
Value []listArtifactsResponseItem `json:"value"`
|
|
}
|
|
listArtifactsResponseItem struct {
|
|
Name string `json:"name"`
|
|
FileContainerResourceURL string `json:"fileContainerResourceUrl"`
|
|
}
|
|
)
|
|
|
|
func (ar artifactRoutes) listArtifacts(ctx *ArtifactContext) {
|
|
_, runID, ok := validateRunID(ctx)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
artifacts, err := actions.ListArtifactsByRunID(ctx, runID)
|
|
if err != nil {
|
|
log.Error("Error getting artifacts: %v", err)
|
|
ctx.Error(http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
if len(artifacts) == 0 {
|
|
log.Debug("[artifact] handleListArtifacts, no artifacts")
|
|
ctx.Error(http.StatusNotFound)
|
|
return
|
|
}
|
|
|
|
var (
|
|
items []listArtifactsResponseItem
|
|
values = make(map[string]bool)
|
|
)
|
|
|
|
for _, art := range artifacts {
|
|
if values[art.ArtifactName] {
|
|
continue
|
|
}
|
|
artifactHash := fmt.Sprintf("%x", md5.Sum([]byte(art.ArtifactName)))
|
|
item := listArtifactsResponseItem{
|
|
Name: art.ArtifactName,
|
|
FileContainerResourceURL: ar.buildArtifactURL(runID, artifactHash, "download_url"),
|
|
}
|
|
items = append(items, item)
|
|
values[art.ArtifactName] = true
|
|
|
|
log.Debug("[artifact] handleListArtifacts, name: %s, url: %s", item.Name, item.FileContainerResourceURL)
|
|
}
|
|
|
|
respData := listArtifactsResponse{
|
|
Count: int64(len(items)),
|
|
Value: items,
|
|
}
|
|
ctx.JSON(http.StatusOK, respData)
|
|
}
|
|
|
|
type (
|
|
downloadArtifactResponse struct {
|
|
Value []downloadArtifactResponseItem `json:"value"`
|
|
}
|
|
downloadArtifactResponseItem struct {
|
|
Path string `json:"path"`
|
|
ItemType string `json:"itemType"`
|
|
ContentLocation string `json:"contentLocation"`
|
|
}
|
|
)
|
|
|
|
// getDownloadArtifactURL generates download url for each artifact
|
|
func (ar artifactRoutes) getDownloadArtifactURL(ctx *ArtifactContext) {
|
|
_, runID, ok := validateRunID(ctx)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
itemPath := util.PathJoinRel(ctx.Req.URL.Query().Get("itemPath"))
|
|
if !validateArtifactHash(ctx, itemPath) {
|
|
return
|
|
}
|
|
|
|
artifacts, err := actions.ListArtifactsByRunIDAndArtifactName(ctx, runID, itemPath)
|
|
if err != nil {
|
|
log.Error("Error getting artifacts: %v", err)
|
|
ctx.Error(http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
if len(artifacts) == 0 {
|
|
log.Debug("[artifact] getDownloadArtifactURL, no artifacts")
|
|
ctx.Error(http.StatusNotFound)
|
|
return
|
|
}
|
|
|
|
if itemPath != artifacts[0].ArtifactName {
|
|
log.Error("Error dismatch artifact name, itemPath: %v, artifact: %v", itemPath, artifacts[0].ArtifactName)
|
|
ctx.Error(http.StatusBadRequest, "Error dismatch artifact name")
|
|
return
|
|
}
|
|
|
|
var items []downloadArtifactResponseItem
|
|
for _, artifact := range artifacts {
|
|
downloadURL := ar.buildArtifactURL(runID, strconv.FormatInt(artifact.ID, 10), "download")
|
|
item := downloadArtifactResponseItem{
|
|
Path: util.PathJoinRel(itemPath, artifact.ArtifactPath),
|
|
ItemType: "file",
|
|
ContentLocation: downloadURL,
|
|
}
|
|
log.Debug("[artifact] getDownloadArtifactURL, path: %s, url: %s", item.Path, item.ContentLocation)
|
|
items = append(items, item)
|
|
}
|
|
respData := downloadArtifactResponse{
|
|
Value: items,
|
|
}
|
|
ctx.JSON(http.StatusOK, respData)
|
|
}
|
|
|
|
// downloadArtifact downloads artifact content
|
|
func (ar artifactRoutes) downloadArtifact(ctx *ArtifactContext) {
|
|
_, runID, ok := validateRunID(ctx)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
artifactID := ctx.ParamsInt64("artifact_id")
|
|
artifact, err := actions.GetArtifactByID(ctx, artifactID)
|
|
if errors.Is(err, util.ErrNotExist) {
|
|
log.Error("Error getting artifact: %v", err)
|
|
ctx.Error(http.StatusNotFound, err.Error())
|
|
return
|
|
} else if err != nil {
|
|
log.Error("Error getting artifact: %v", err)
|
|
ctx.Error(http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
if artifact.RunID != runID {
|
|
log.Error("Error dismatch runID and artifactID, task: %v, artifact: %v", runID, artifactID)
|
|
ctx.Error(http.StatusBadRequest, err.Error())
|
|
return
|
|
}
|
|
|
|
fd, err := ar.fs.Open(artifact.StoragePath)
|
|
if err != nil {
|
|
log.Error("Error opening file: %v", err)
|
|
ctx.Error(http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
defer fd.Close()
|
|
|
|
// if artifact is compressed, set content-encoding header to gzip
|
|
if artifact.ContentEncoding == "gzip" {
|
|
ctx.Resp.Header().Set("Content-Encoding", "gzip")
|
|
}
|
|
log.Debug("[artifact] downloadArtifact, name: %s, path: %s, storage: %s, size: %d", artifact.ArtifactName, artifact.ArtifactPath, artifact.StoragePath, artifact.FileSize)
|
|
ctx.ServeContent(fd, &context.ServeHeaderOptions{
|
|
Filename: artifact.ArtifactName,
|
|
LastModified: artifact.CreatedUnix.AsLocalTime(),
|
|
})
|
|
}
|