From bcb52aef09160c1057baa97b9275e8fb63587b12 Mon Sep 17 00:00:00 2001 From: guillep2k <18600385+guillep2k@users.noreply.github.com> Date: Sat, 1 Feb 2020 23:17:44 -0300 Subject: [PATCH] Implement "embedded" command to extract static resources (#9982) * draft * Implement extract command * Fix nits and force args on extract * Add !bindata stub, support Windows, fmt * fix vendored flag * Remove leading slash for matching * Add docs * Fix typos * Add embedded view command Co-authored-by: zeripath Co-authored-by: Lunny Xiao --- cmd/embedded.go | 332 ++++++++++++++++++ cmd/embedded_stub.go | 30 ++ .../doc/advanced/cmd-embedded.en-us.md | 115 ++++++ .../doc/advanced/customizing-gitea.en-us.md | 19 +- main.go | 1 + modules/options/static.go | 31 ++ modules/public/static.go | 33 ++ modules/templates/static.go | 13 + 8 files changed, 568 insertions(+), 6 deletions(-) create mode 100644 cmd/embedded.go create mode 100644 cmd/embedded_stub.go create mode 100644 docs/content/doc/advanced/cmd-embedded.en-us.md diff --git a/cmd/embedded.go b/cmd/embedded.go new file mode 100644 index 0000000000..363b85c066 --- /dev/null +++ b/cmd/embedded.go @@ -0,0 +1,332 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build bindata + +package cmd + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/options" + "code.gitea.io/gitea/modules/public" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/templates" + + "github.com/gobwas/glob" + "github.com/urfave/cli" +) + +// Cmdembedded represents the available extract sub-command. +var ( + Cmdembedded = cli.Command{ + Name: "embedded", + Usage: "Extract embedded resources", + Description: "A command for extracting embedded resources, like templates and images", + Subcommands: []cli.Command{ + subcmdList, + subcmdView, + subcmdExtract, + }, + } + + subcmdList = cli.Command{ + Name: "list", + Usage: "List files matching the given pattern", + Action: runList, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "include-vendored,vendor", + Usage: "Include files under public/vendor as well", + }, + }, + } + + subcmdView = cli.Command{ + Name: "view", + Usage: "View a file matching the given pattern", + Action: runView, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "include-vendored,vendor", + Usage: "Include files under public/vendor as well", + }, + }, + } + + subcmdExtract = cli.Command{ + Name: "extract", + Usage: "Extract resources", + Action: runExtract, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "include-vendored,vendor", + Usage: "Include files under public/vendor as well", + }, + cli.BoolFlag{ + Name: "overwrite", + Usage: "Overwrite files if they already exist", + }, + cli.BoolFlag{ + Name: "rename", + Usage: "Rename files as {name}.bak if they already exist (overwrites previous .bak)", + }, + cli.BoolFlag{ + Name: "custom", + Usage: "Extract to the 'custom' directory as per app.ini", + }, + cli.StringFlag{ + Name: "destination,dest-dir", + Usage: "Extract to the specified directory", + }, + }, + } + + sections map[string]*section + assets []asset +) + +type section struct { + Path string + Names func() []string + IsDir func(string) (bool, error) + Asset func(string) ([]byte, error) +} + +type asset struct { + Section *section + Name string + Path string +} + +func initEmbeddedExtractor(c *cli.Context) error { + + // Silence the console logger + log.DelNamedLogger("console") + log.DelNamedLogger(log.DEFAULT) + + // Read configuration file + setting.NewContext() + + pats, err := getPatterns(c.Args()) + if err != nil { + return err + } + sections := make(map[string]*section, 3) + + sections["public"] = §ion{Path: "public", Names: public.AssetNames, IsDir: public.AssetIsDir, Asset: public.Asset} + sections["options"] = §ion{Path: "options", Names: options.AssetNames, IsDir: options.AssetIsDir, Asset: options.Asset} + sections["templates"] = §ion{Path: "templates", Names: templates.AssetNames, IsDir: templates.AssetIsDir, Asset: templates.Asset} + + for _, sec := range sections { + assets = append(assets, buildAssetList(sec, pats, c)...) + } + + // Sort assets + sort.SliceStable(assets, func(i, j int) bool { + return assets[i].Path < assets[j].Path + }) + + return nil +} + +func runList(c *cli.Context) error { + if err := runListDo(c); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + return err + } + return nil +} + +func runView(c *cli.Context) error { + if err := runViewDo(c); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + return err + } + return nil +} + +func runExtract(c *cli.Context) error { + if err := runExtractDo(c); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + return err + } + return nil +} + +func runListDo(c *cli.Context) error { + if err := initEmbeddedExtractor(c); err != nil { + return err + } + + for _, a := range assets { + fmt.Println(a.Path) + } + + return nil +} + +func runViewDo(c *cli.Context) error { + if err := initEmbeddedExtractor(c); err != nil { + return err + } + + if len(assets) == 0 { + return fmt.Errorf("No files matched the given pattern") + } else if len(assets) > 1 { + return fmt.Errorf("Too many files matched the given pattern; try to be more specific") + } + + data, err := assets[0].Section.Asset(assets[0].Name) + if err != nil { + return fmt.Errorf("%s: %v", assets[0].Path, err) + } + + if _, err = os.Stdout.Write(data); err != nil { + return fmt.Errorf("%s: %v", assets[0].Path, err) + } + + return nil +} + +func runExtractDo(c *cli.Context) error { + if err := initEmbeddedExtractor(c); err != nil { + return err + } + + if len(c.Args()) == 0 { + return fmt.Errorf("A list of pattern of files to extract is mandatory (e.g. '**' for all)") + } + + destdir := "." + + if c.IsSet("destination") { + destdir = c.String("destination") + } else if c.Bool("custom") { + destdir = setting.CustomPath + fmt.Println("Using app.ini at", setting.CustomConf) + } + + fi, err := os.Stat(destdir) + if errors.Is(err, os.ErrNotExist) { + // In case Windows users attempt to provide a forward-slash path + wdestdir := filepath.FromSlash(destdir) + if wfi, werr := os.Stat(wdestdir); werr == nil { + destdir = wdestdir + fi = wfi + err = nil + } + } + if err != nil { + return fmt.Errorf("%s: %s", destdir, err) + } else if !fi.IsDir() { + return fmt.Errorf("%s is not a directory.", destdir) + } + + fmt.Printf("Extracting to %s:\n", destdir) + + overwrite := c.Bool("overwrite") + rename := c.Bool("rename") + + for _, a := range assets { + if err := extractAsset(destdir, a, overwrite, rename); err != nil { + // Non-fatal error + fmt.Fprintf(os.Stderr, "%s: %v", a.Path, err) + } + } + + return nil +} + +func extractAsset(d string, a asset, overwrite, rename bool) error { + dest := filepath.Join(d, filepath.FromSlash(a.Path)) + dir := filepath.Dir(dest) + + data, err := a.Section.Asset(a.Name) + if err != nil { + return fmt.Errorf("%s: %v", a.Path, err) + } + + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return fmt.Errorf("%s: %v", dir, err) + } + + perms := os.ModePerm & 0666 + + fi, err := os.Lstat(dest) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("%s: %v", dest, err) + } + } else if !overwrite && !rename { + fmt.Printf("%s already exists; skipped.\n", dest) + return nil + } else if !fi.Mode().IsRegular() { + return fmt.Errorf("%s already exists, but it's not a regular file", dest) + } else if rename { + if err := os.Rename(dest, dest+".bak"); err != nil { + return fmt.Errorf("Error creating backup for %s: %v", dest, err) + } + // Attempt to respect file permissions mask (even if user:group will be set anew) + perms = fi.Mode() + } + + file, err := os.OpenFile(dest, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, perms) + if err != nil { + return fmt.Errorf("%s: %v", dest, err) + } + defer file.Close() + + if _, err = file.Write(data); err != nil { + return fmt.Errorf("%s: %v", dest, err) + } + + fmt.Println(dest) + + return nil +} + +func buildAssetList(sec *section, globs []glob.Glob, c *cli.Context) []asset { + var results = make([]asset, 0, 64) + for _, name := range sec.Names() { + if isdir, err := sec.IsDir(name); !isdir && err == nil { + if sec.Path == "public" && + strings.HasPrefix(name, "vendor/") && + !c.Bool("include-vendored") { + continue + } + matchName := sec.Path + "/" + name + for _, g := range globs { + if g.Match(matchName) { + results = append(results, asset{Section: sec, + Name: name, + Path: sec.Path + "/" + name}) + break + } + } + } + } + return results +} + +func getPatterns(args []string) ([]glob.Glob, error) { + if len(args) == 0 { + args = []string{"**"} + } + pat := make([]glob.Glob, len(args)) + for i := range args { + if g, err := glob.Compile(args[i], '/'); err != nil { + return nil, fmt.Errorf("'%s': Invalid glob pattern: %v", args[i], err) + } else { + pat[i] = g + } + } + return pat, nil +} diff --git a/cmd/embedded_stub.go b/cmd/embedded_stub.go new file mode 100644 index 0000000000..1f9af7b86b --- /dev/null +++ b/cmd/embedded_stub.go @@ -0,0 +1,30 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !bindata + +package cmd + +import ( + "fmt" + "os" + + "github.com/urfave/cli" +) + +// Cmdembedded represents the available extract sub-command. +var ( + Cmdembedded = cli.Command{ + Name: "embedded", + Usage: "Extract embedded resources", + Description: "A command for extracting embedded resources, like templates and images", + Action: extractorNotImplemented, + } +) + +func extractorNotImplemented(c *cli.Context) error { + err := fmt.Errorf("Sorry: the 'embedded' subcommand is not available in builds without bindata") + fmt.Fprintf(os.Stderr, "%s\n", err) + return err +} diff --git a/docs/content/doc/advanced/cmd-embedded.en-us.md b/docs/content/doc/advanced/cmd-embedded.en-us.md new file mode 100644 index 0000000000..e036d17498 --- /dev/null +++ b/docs/content/doc/advanced/cmd-embedded.en-us.md @@ -0,0 +1,115 @@ +--- +date: "2020-01-25T21:00:00-03:00" +title: "Embedded data extraction tool" +slug: "cmd-embedded" +weight: 40 +toc: true +draft: false +menu: + sidebar: + parent: "advanced" + name: "Embedded data extraction tool" + weight: 40 + identifier: "cmd-embedded" +--- + +# Embedded data extraction tool + +Gitea's executable contains all the resources required to run: templates, images, style-sheets +and translations. Any of them can be overridden by placing a replacement in a matching path +inside the `custom` directory (see [Customizing Gitea]({{< relref "doc/advanced/customizing-gitea.en-us.md" >}})). + +To obtain a copy of the embedded resources ready for editing, the `embedded` command from the CLI +can be used from the OS shell interface. + +## Listing resources + +To list resources embedded in Gitea's executable, use the following syntax: + +``` +gitea embedded list [--include-vendored] [patterns...] +``` + +The `--include-vendored` flag makes the command include vendored files, which are +normally excluded; that is, files from external libraries that are required for Gitea +(e.g. [font-awesome](https://fontawesome.com/), [octicons](https://octicons.github.com/), etc). + +A list of file search patterns can be provided. Gitea uses [gobwas/glob](https://github.com/gobwas/glob) +for its glob syntax. Here are some examples: + +- List all template files, in any virtual directory: `**.tmpl` +- List all mail template files: `templates/mail/**.tmpl` +- List all files inside `public/img`: `public/img/**` + +Don't forget to use quotes for the patterns, as spaces, `*` and other characters might have +a special meaning for your command shell. + +If no pattern is provided, all files are listed. + +#### Example + +Listing all embedded files with `openid` in their path: + +``` +$ gitea embedded list '**openid**' +public/img/auth/openid_connect.png +public/img/openid-16x16.png +templates/user/auth/finalize_openid.tmpl +templates/user/auth/signin_openid.tmpl +templates/user/auth/signup_openid_connect.tmpl +templates/user/auth/signup_openid_navbar.tmpl +templates/user/auth/signup_openid_register.tmpl +templates/user/settings/security_openid.tmpl +``` + +## Extracting resources + +To extract resources embedded in Gitea's executable, use the following syntax: + +``` +gitea [--config {file}] embedded extract [--destination {dir}|--custom] [--overwrite|--rename] [--include-vendored] {patterns...} +``` + +The `--config` option tells gitea the location of the `app.ini` configuration file if +it's not in its default location. This option is only used with the `--custom` flag. + +The `--destination` option tells gitea the directory where the files must be extracted to. +The default is the current directory. + +The `--custom` flag tells gitea to extract the files directly into the `custom` directory. +For this to work, the command needs to know the location of the `app.ini` configuration +file (`--config`) and, depending of the configuration, be ran from the directory where +gitea normally starts. See [Customizing Gitea]({{< relref "doc/advanced/customizing-gitea.en-us.md" >}}) for details. + +The `--overwrite` flag allows any existing files in the destination directory to be overwritten. + +The `--rename` flag tells gitea to rename any existing files in the destination directory +as `filename.bak`. Previous `.bak` files are overwritten. + +At least one file search pattern must be provided; see `list` subcomand above for pattern +syntax and examples. + +#### Important notice + +Make sure to **only extract those files that require customization**. Files that +are present in the `custom` directory are not upgraded by Gitea's upgrade process. +When Gitea is upgraded to a new version (by replacing the executable), many of the +embedded files will suffer changes. Gitea will honor and use any files found +in the `custom` directory, even if they are old and incompatible. + +#### Example + +Extracting mail templates to a temporary directory: + +``` +$ mkdir tempdir +$ gitea embedded extract --destination tempdir 'templates/mail/**.tmpl' +Extracting to tempdir: +tempdir/templates/mail/auth/activate.tmpl +tempdir/templates/mail/auth/activate_email.tmpl +tempdir/templates/mail/auth/register_notify.tmpl +tempdir/templates/mail/auth/reset_passwd.tmpl +tempdir/templates/mail/issue/assigned.tmpl +tempdir/templates/mail/issue/default.tmpl +tempdir/templates/mail/notify/collaborator.tmpl +``` diff --git a/docs/content/doc/advanced/customizing-gitea.en-us.md b/docs/content/doc/advanced/customizing-gitea.en-us.md index f42ade7799..73127dd81a 100644 --- a/docs/content/doc/advanced/customizing-gitea.en-us.md +++ b/docs/content/doc/advanced/customizing-gitea.en-us.md @@ -57,14 +57,21 @@ the url `http://gitea.domain.tld/image.png`. Place the png image at the following path: `custom/public/img/avatar_default.png` -## Customizing Gitea pages +## Customizing Gitea pages and resources -The `custom/templates` folder allows changing every single page of Gitea. Templates -to override can be found in the [`templates`](https://github.com/go-gitea/gitea/tree/master/templates) directory of Gitea source (Note: the example link is from `master` branch. Make sure to copy templates from same release you are using). Override by -making a copy of the file under `custom/templates` using a full path structure -matching source. +Gitea's executable contains all the resources required to run: templates, images, style-sheets +and translations. Any of them can be overridden by placing a replacement in a matching path +inside the `custom` directory. For example, to replace the default `.gitignore` provided +for C++ repositories, we want to replace `options/gitignore/C++`. To do this, a replacement +must be placed in `custom/options/gitignore/C++` (see about the location of the `custom` +directory at the top of this document). -Any statement contained inside `{{` and `}}` are Gitea's template syntax and +Every single page of Gitea can be changed. Dynamic content is generated using [go templates](https://golang.org/pkg/html/template/), +which can be modified by placing replacements below the `custom/templates` directory. + +To obtain any embedded file (including templates), the [`gitea embedded` tool]({{< relref "doc/advanced/cmd-embedded.en-us.md" >}}) can be used. Alternatively, they can be found in the [`templates`](https://github.com/go-gitea/gitea/tree/master/templates) directory of Gitea source (Note: the example link is from the `master` branch. Make sure to use templates compatible with the release you are using). + +Be aware that any statement contained inside `{{` and `}}` are Gitea's template syntax and shouldn't be touched without fully understanding these components. ### Customizing startpage / homepage diff --git a/main.go b/main.go index c97513293a..bf7c59c252 100644 --- a/main.go +++ b/main.go @@ -70,6 +70,7 @@ arguments - which can alternatively be run by running the subcommand web.` cmd.CmdConvert, cmd.CmdDoctor, cmd.CmdManager, + cmd.Cmdembedded, } // Now adjust these commands to add our global configuration options diff --git a/modules/options/static.go b/modules/options/static.go index f8811e43ac..39f56f42f4 100644 --- a/modules/options/static.go +++ b/modules/options/static.go @@ -113,6 +113,37 @@ func fileFromDir(name string) ([]byte, error) { return ioutil.ReadAll(f) } +func Asset(name string) ([]byte, error) { + f, err := Assets.Open("/" + name) + if err != nil { + return nil, err + } + defer f.Close() + return ioutil.ReadAll(f) +} + +func AssetNames() []string { + realFS := Assets.(vfsgen۰FS) + var results = make([]string, 0, len(realFS)) + for k := range realFS { + results = append(results, k[1:]) + } + return results +} + +func AssetIsDir(name string) (bool, error) { + if f, err := Assets.Open("/" + name); err != nil { + return false, err + } else { + defer f.Close() + if fi, err := f.Stat(); err != nil { + return false, err + } else { + return fi.IsDir(), nil + } + } +} + // IsDynamic will return false when using embedded data (-tags bindata) func IsDynamic() bool { return false diff --git a/modules/public/static.go b/modules/public/static.go index 3e8865c3f9..76050632c9 100644 --- a/modules/public/static.go +++ b/modules/public/static.go @@ -7,6 +7,8 @@ package public import ( + "io/ioutil" + "gitea.com/macaron/macaron" ) @@ -17,3 +19,34 @@ func Static(opts *Options) macaron.Handler { // used when in the options there is no FileSystem. return opts.staticHandler("") } + +func Asset(name string) ([]byte, error) { + f, err := Assets.Open("/" + name) + if err != nil { + return nil, err + } + defer f.Close() + return ioutil.ReadAll(f) +} + +func AssetNames() []string { + realFS := Assets.(vfsgen۰FS) + var results = make([]string, 0, len(realFS)) + for k := range realFS { + results = append(results, k[1:]) + } + return results +} + +func AssetIsDir(name string) (bool, error) { + if f, err := Assets.Open("/" + name); err != nil { + return false, err + } else { + defer f.Close() + if fi, err := f.Stat(); err != nil { + return false, err + } else { + return fi.IsDir(), nil + } + } +} diff --git a/modules/templates/static.go b/modules/templates/static.go index 435ccb1f95..5bc4e33e1c 100644 --- a/modules/templates/static.go +++ b/modules/templates/static.go @@ -229,3 +229,16 @@ func AssetNames() []string { } return results } + +func AssetIsDir(name string) (bool, error) { + if f, err := Assets.Open("/" + name); err != nil { + return false, err + } else { + defer f.Close() + if fi, err := f.Stat(); err != nil { + return false, err + } else { + return fi.IsDir(), nil + } + } +}