1
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2024-12-04 10:30:19 -05:00
forgejo/modules/markup/html.go

1281 lines
36 KiB
Go
Raw Normal View History

// Copyright 2017 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package markup
import (
"bytes"
"io"
"net/url"
"path"
"path/filepath"
"regexp"
"strings"
"sync"
"code.gitea.io/gitea/modules/base"
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
"code.gitea.io/gitea/modules/emoji"
Check commit message hashes before making links (#7713) * Check commit message hashes before making links Previously, when formatting commit messages, anything that looked like SHA1 hashes was turned into a link using regex. This meant that certain phrases or numbers such as `777777` or `deadbeef` could be recognized as a commit even if the repository has no commit with those hashes. This change will make it so that anything that looks like a SHA1 hash using regex will then also be checked to ensure that there is a commit in the repository with that hash before making a link. Signed-off-by: Gary Kim <gary@garykim.dev> * Use gogit to check if commit exists This commit modifies the commit hash check in the render for commit messages to use gogit for better performance. Signed-off-by: Gary Kim <gary@garykim.dev> * Make code cleaner Signed-off-by: Gary Kim <gary@garykim.dev> * Use rev-parse to check if commit exists Signed-off-by: Gary Kim <gary@garykim.dev> * Add and modify tests for checking hashes in html link rendering Signed-off-by: Gary Kim <gary@garykim.dev> * Return error in sha1CurrentPatternProcessor Co-Authored-By: mrsdizzie <info@mrsdizzie.com> * Import Gitea log module Signed-off-by: Gary Kim <gary@garykim.dev> * Revert "Return error in sha1CurrentPatternProcessor" This reverts commit 28f561cac46ef7e51aa26aefcbe9aca4671366a6. Signed-off-by: Gary Kim <gary@garykim.dev> * Add debug logging to sha1CurrentPatternProcessor This will log errors by the git command run in sha1CurrentPatternProcessor if the error is one that was unexpected. Signed-off-by: Gary Kim <gary@garykim.dev>
2019-08-14 04:04:55 -04:00
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/markup/common"
"code.gitea.io/gitea/modules/references"
"code.gitea.io/gitea/modules/regexplru"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/templates/vars"
Append `(comment)` when a link points at a comment rather than the whole issue (#23734) Close #23671 For the feature mentioned above, this PR append ' (comment)' to the rendered html if it is a hashcomment. After the PR, type in the following ``` pull request from other repo: http://localhost:3000/testOrg/testOrgRepo/pulls/2 pull request from this repo: http://localhost:3000/aaa/testA/pulls/2 issue comment from this repo: http://localhost:3000/aaa/testA/issues/1#issuecomment-18 http://localhost:3000/aaa/testA/pulls/2#issue-9 issue comment from other repo: http://localhost:3000/testOrg/testOrgRepo/pulls/2#issuecomment-24 http://localhost:3000/testOrg/testOrgRepo/pulls/2#issue ``` Gives: <img width="687" alt="截屏2023-03-27 13 53 06" src="https://user-images.githubusercontent.com/17645053/227852387-2b218e0d-3468-4d90-ad81-d702ddd17fd2.png"> Other than the above feature, this PR also includes two other changes: 1 Right now, the render of links from file changed tab in pull request might not be very proper, for example, if type in the following. (not sure if this is an issue or design, if not an issue, I will revert the changes). example on [try.gitea.io](https://try.gitea.io/HesterG/testrepo/pulls/1) ``` https://try.gitea.io/HesterG/testrepo/pulls/1/files#issuecomment-162725 https://try.gitea.io/HesterG/testrepo/pulls/1/files ``` it will render the following <img width="899" alt="截屏2023-03-24 15 41 37" src="https://user-images.githubusercontent.com/17645053/227456117-5eccedb7-9118-4540-929d-aee9a76de852.png"> In this PR, skip processing the link into a ref issue if it is a link from files changed tab in pull request After: type in following ``` hash comment on files changed tab: http://localhost:3000/testOrg/testOrgRepo/pulls/2/files#issuecomment-24 files changed link: http://localhost:3000/testOrg/testOrgRepo/pulls/2/files ``` Gives <img width="708" alt="截屏2023-03-27 22 09 02" src="https://user-images.githubusercontent.com/17645053/227964273-5dc06c50-3713-489c-b05d-d95367d0ab0f.png"> 2 Right now, after editing the comment area, there will not be tippys attached to `ref-issue`; and no tippy attached on preview as well. example: https://user-images.githubusercontent.com/17645053/227850540-5ae34e2d-b1d7-4d0d-9726-7701bf825d1f.mov In this PR, in frontend, make sure tippy is added after editing the comment, and to the comment on preview tab After: https://user-images.githubusercontent.com/17645053/227853777-06f56b4c-1148-467c-b6f7-f79418e67504.mov
2023-04-03 04:02:57 -04:00
"code.gitea.io/gitea/modules/translation"
"code.gitea.io/gitea/modules/util"
"golang.org/x/net/html"
"golang.org/x/net/html/atom"
"mvdan.cc/xurls/v2"
)
// Issue name styles
const (
IssueNameStyleNumeric = "numeric"
IssueNameStyleAlphanumeric = "alphanumeric"
IssueNameStyleRegexp = "regexp"
)
var (
// NOTE: All below regex matching do not perform any extra validation.
// Thus a link is produced even if the linked entity does not exist.
// While fast, this is also incorrect and lead to false positives.
// TODO: fix invalid linking issue
// valid chars in encoded path and parameter: [-+~_%.a-zA-Z0-9/]
// sha1CurrentPattern matches string that represents a commit SHA, e.g. d8a994ef243349f321568f9e36d5c3f444b99cae
// Although SHA1 hashes are 40 chars long, the regex matches the hash from 7 to 40 chars in length
// so that abbreviated hash links can be used as well. This matches git and GitHub usability.
sha1CurrentPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([0-9a-f]{7,40})(?:\s|$|\)|\]|[.,](\s|$))`)
// shortLinkPattern matches short but difficult to parse [[name|link|arg=test]] syntax
shortLinkPattern = regexp.MustCompile(`\[\[(.*?)\]\](\w*)`)
// anySHA1Pattern splits url containing SHA into parts
anySHA1Pattern = regexp.MustCompile(`https?://(?:\S+/){4,5}([0-9a-f]{40})(/[-+~_%.a-zA-Z0-9/]+)?(#[-+~_%.a-zA-Z0-9]+)?`)
// comparePattern matches "http://domain/org/repo/compare/COMMIT1...COMMIT2#hash"
comparePattern = regexp.MustCompile(`https?://(?:\S+/){4,5}([0-9a-f]{7,40})(\.\.\.?)([0-9a-f]{7,40})?(#[-+~_%.a-zA-Z0-9]+)?`)
validLinksPattern = regexp.MustCompile(`^[a-z][\w-]+://`)
// While this email regex is definitely not perfect and I'm sure you can come up
// with edge cases, it is still accepted by the CommonMark specification, as
// well as the HTML5 spec:
// http://spec.commonmark.org/0.28/#email-address
// https://html.spec.whatwg.org/multipage/input.html#e-mail-state-(type%3Demail)
emailRegex = regexp.MustCompile("(?:\\s|^|\\(|\\[)([a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9]{2,}(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+)(?:\\s|$|\\)|\\]|\\.(\\s|$))")
// blackfriday extensions create IDs like fn:user-content-footnote
blackfridayExtRegex = regexp.MustCompile(`[^:]*:user-content-`)
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
// EmojiShortCodeRegex find emoji by alias like :smile:
EmojiShortCodeRegex = regexp.MustCompile(`:[-+\w]+:`)
)
// CSS class for action keywords (e.g. "closes: #1")
const keywordClass = "issue-keyword"
// IsLink reports whether link fits valid format.
func IsLink(link []byte) bool {
return isLink(link)
}
// isLink reports whether link fits valid format.
func isLink(link []byte) bool {
return validLinksPattern.Match(link)
}
func isLinkStr(link string) bool {
return validLinksPattern.MatchString(link)
}
// regexp for full links to issues/pulls
var issueFullPattern *regexp.Regexp
// Once for to prevent races
var issueFullPatternOnce sync.Once
Append `(comment)` when a link points at a comment rather than the whole issue (#23734) Close #23671 For the feature mentioned above, this PR append ' (comment)' to the rendered html if it is a hashcomment. After the PR, type in the following ``` pull request from other repo: http://localhost:3000/testOrg/testOrgRepo/pulls/2 pull request from this repo: http://localhost:3000/aaa/testA/pulls/2 issue comment from this repo: http://localhost:3000/aaa/testA/issues/1#issuecomment-18 http://localhost:3000/aaa/testA/pulls/2#issue-9 issue comment from other repo: http://localhost:3000/testOrg/testOrgRepo/pulls/2#issuecomment-24 http://localhost:3000/testOrg/testOrgRepo/pulls/2#issue ``` Gives: <img width="687" alt="截屏2023-03-27 13 53 06" src="https://user-images.githubusercontent.com/17645053/227852387-2b218e0d-3468-4d90-ad81-d702ddd17fd2.png"> Other than the above feature, this PR also includes two other changes: 1 Right now, the render of links from file changed tab in pull request might not be very proper, for example, if type in the following. (not sure if this is an issue or design, if not an issue, I will revert the changes). example on [try.gitea.io](https://try.gitea.io/HesterG/testrepo/pulls/1) ``` https://try.gitea.io/HesterG/testrepo/pulls/1/files#issuecomment-162725 https://try.gitea.io/HesterG/testrepo/pulls/1/files ``` it will render the following <img width="899" alt="截屏2023-03-24 15 41 37" src="https://user-images.githubusercontent.com/17645053/227456117-5eccedb7-9118-4540-929d-aee9a76de852.png"> In this PR, skip processing the link into a ref issue if it is a link from files changed tab in pull request After: type in following ``` hash comment on files changed tab: http://localhost:3000/testOrg/testOrgRepo/pulls/2/files#issuecomment-24 files changed link: http://localhost:3000/testOrg/testOrgRepo/pulls/2/files ``` Gives <img width="708" alt="截屏2023-03-27 22 09 02" src="https://user-images.githubusercontent.com/17645053/227964273-5dc06c50-3713-489c-b05d-d95367d0ab0f.png"> 2 Right now, after editing the comment area, there will not be tippys attached to `ref-issue`; and no tippy attached on preview as well. example: https://user-images.githubusercontent.com/17645053/227850540-5ae34e2d-b1d7-4d0d-9726-7701bf825d1f.mov In this PR, in frontend, make sure tippy is added after editing the comment, and to the comment on preview tab After: https://user-images.githubusercontent.com/17645053/227853777-06f56b4c-1148-467c-b6f7-f79418e67504.mov
2023-04-03 04:02:57 -04:00
// regexp for full links to hash comment in pull request files changed tab
var filesChangedFullPattern *regexp.Regexp
// Once for to prevent races
var filesChangedFullPatternOnce sync.Once
func getIssueFullPattern() *regexp.Regexp {
issueFullPatternOnce.Do(func() {
Append `(comment)` when a link points at a comment rather than the whole issue (#23734) Close #23671 For the feature mentioned above, this PR append ' (comment)' to the rendered html if it is a hashcomment. After the PR, type in the following ``` pull request from other repo: http://localhost:3000/testOrg/testOrgRepo/pulls/2 pull request from this repo: http://localhost:3000/aaa/testA/pulls/2 issue comment from this repo: http://localhost:3000/aaa/testA/issues/1#issuecomment-18 http://localhost:3000/aaa/testA/pulls/2#issue-9 issue comment from other repo: http://localhost:3000/testOrg/testOrgRepo/pulls/2#issuecomment-24 http://localhost:3000/testOrg/testOrgRepo/pulls/2#issue ``` Gives: <img width="687" alt="截屏2023-03-27 13 53 06" src="https://user-images.githubusercontent.com/17645053/227852387-2b218e0d-3468-4d90-ad81-d702ddd17fd2.png"> Other than the above feature, this PR also includes two other changes: 1 Right now, the render of links from file changed tab in pull request might not be very proper, for example, if type in the following. (not sure if this is an issue or design, if not an issue, I will revert the changes). example on [try.gitea.io](https://try.gitea.io/HesterG/testrepo/pulls/1) ``` https://try.gitea.io/HesterG/testrepo/pulls/1/files#issuecomment-162725 https://try.gitea.io/HesterG/testrepo/pulls/1/files ``` it will render the following <img width="899" alt="截屏2023-03-24 15 41 37" src="https://user-images.githubusercontent.com/17645053/227456117-5eccedb7-9118-4540-929d-aee9a76de852.png"> In this PR, skip processing the link into a ref issue if it is a link from files changed tab in pull request After: type in following ``` hash comment on files changed tab: http://localhost:3000/testOrg/testOrgRepo/pulls/2/files#issuecomment-24 files changed link: http://localhost:3000/testOrg/testOrgRepo/pulls/2/files ``` Gives <img width="708" alt="截屏2023-03-27 22 09 02" src="https://user-images.githubusercontent.com/17645053/227964273-5dc06c50-3713-489c-b05d-d95367d0ab0f.png"> 2 Right now, after editing the comment area, there will not be tippys attached to `ref-issue`; and no tippy attached on preview as well. example: https://user-images.githubusercontent.com/17645053/227850540-5ae34e2d-b1d7-4d0d-9726-7701bf825d1f.mov In this PR, in frontend, make sure tippy is added after editing the comment, and to the comment on preview tab After: https://user-images.githubusercontent.com/17645053/227853777-06f56b4c-1148-467c-b6f7-f79418e67504.mov
2023-04-03 04:02:57 -04:00
// example: https://domain/org/repo/pulls/27#hash
issueFullPattern = regexp.MustCompile(regexp.QuoteMeta(setting.AppURL) +
`[\w_.-]+/[\w_.-]+/(?:issues|pulls)/((?:\w{1,10}-)?[1-9][0-9]*)([\?|#](\S+)?)?\b`)
})
return issueFullPattern
}
Append `(comment)` when a link points at a comment rather than the whole issue (#23734) Close #23671 For the feature mentioned above, this PR append ' (comment)' to the rendered html if it is a hashcomment. After the PR, type in the following ``` pull request from other repo: http://localhost:3000/testOrg/testOrgRepo/pulls/2 pull request from this repo: http://localhost:3000/aaa/testA/pulls/2 issue comment from this repo: http://localhost:3000/aaa/testA/issues/1#issuecomment-18 http://localhost:3000/aaa/testA/pulls/2#issue-9 issue comment from other repo: http://localhost:3000/testOrg/testOrgRepo/pulls/2#issuecomment-24 http://localhost:3000/testOrg/testOrgRepo/pulls/2#issue ``` Gives: <img width="687" alt="截屏2023-03-27 13 53 06" src="https://user-images.githubusercontent.com/17645053/227852387-2b218e0d-3468-4d90-ad81-d702ddd17fd2.png"> Other than the above feature, this PR also includes two other changes: 1 Right now, the render of links from file changed tab in pull request might not be very proper, for example, if type in the following. (not sure if this is an issue or design, if not an issue, I will revert the changes). example on [try.gitea.io](https://try.gitea.io/HesterG/testrepo/pulls/1) ``` https://try.gitea.io/HesterG/testrepo/pulls/1/files#issuecomment-162725 https://try.gitea.io/HesterG/testrepo/pulls/1/files ``` it will render the following <img width="899" alt="截屏2023-03-24 15 41 37" src="https://user-images.githubusercontent.com/17645053/227456117-5eccedb7-9118-4540-929d-aee9a76de852.png"> In this PR, skip processing the link into a ref issue if it is a link from files changed tab in pull request After: type in following ``` hash comment on files changed tab: http://localhost:3000/testOrg/testOrgRepo/pulls/2/files#issuecomment-24 files changed link: http://localhost:3000/testOrg/testOrgRepo/pulls/2/files ``` Gives <img width="708" alt="截屏2023-03-27 22 09 02" src="https://user-images.githubusercontent.com/17645053/227964273-5dc06c50-3713-489c-b05d-d95367d0ab0f.png"> 2 Right now, after editing the comment area, there will not be tippys attached to `ref-issue`; and no tippy attached on preview as well. example: https://user-images.githubusercontent.com/17645053/227850540-5ae34e2d-b1d7-4d0d-9726-7701bf825d1f.mov In this PR, in frontend, make sure tippy is added after editing the comment, and to the comment on preview tab After: https://user-images.githubusercontent.com/17645053/227853777-06f56b4c-1148-467c-b6f7-f79418e67504.mov
2023-04-03 04:02:57 -04:00
func getFilesChangedFullPattern() *regexp.Regexp {
filesChangedFullPatternOnce.Do(func() {
// example: https://domain/org/repo/pulls/27/files#hash
filesChangedFullPattern = regexp.MustCompile(regexp.QuoteMeta(setting.AppURL) +
`[\w_.-]+/[\w_.-]+/pulls/((?:\w{1,10}-)?[1-9][0-9]*)/files([\?|#](\S+)?)?\b`)
})
return filesChangedFullPattern
}
// CustomLinkURLSchemes allows for additional schemes to be detected when parsing links within text
func CustomLinkURLSchemes(schemes []string) {
schemes = append(schemes, "http", "https")
withAuth := make([]string, 0, len(schemes))
validScheme := regexp.MustCompile(`^[a-z]+$`)
for _, s := range schemes {
if !validScheme.MatchString(s) {
continue
}
without := false
for _, sna := range xurls.SchemesNoAuthority {
if s == sna {
without = true
break
}
}
if without {
s += ":"
} else {
s += "://"
}
withAuth = append(withAuth, s)
}
common.LinkRegex, _ = xurls.StrictMatchingScheme(strings.Join(withAuth, "|"))
}
// IsSameDomain checks if given url string has the same hostname as current Gitea instance
func IsSameDomain(s string) bool {
if strings.HasPrefix(s, "/") {
return true
}
if uapp, err := url.Parse(setting.AppURL); err == nil {
if u, err := url.Parse(s); err == nil {
return u.Host == uapp.Host
}
return false
}
return false
}
type postProcessError struct {
context string
err error
}
func (p *postProcessError) Error() string {
2019-06-12 15:41:28 -04:00
return "PostProcess: " + p.context + ", " + p.err.Error()
}
type processor func(ctx *RenderContext, node *html.Node)
var defaultProcessors = []processor{
fullIssuePatternProcessor,
comparePatternProcessor,
fullSha1PatternProcessor,
shortLinkProcessor,
linkProcessor,
mentionProcessor,
issueIndexPatternProcessor,
commitCrossReferencePatternProcessor,
sha1CurrentPatternProcessor,
emailAddressProcessor,
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
emojiProcessor,
emojiShortCodeProcessor,
}
// PostProcess does the final required transformations to the passed raw HTML
// data, and ensures its validity. Transformations include: replacing links and
// emails with HTML links, parsing shortlinks in the format of [[Link]], like
// MediaWiki, linking issues in the format #ID, and mentions in the format
// @user, and others.
func PostProcess(
ctx *RenderContext,
input io.Reader,
output io.Writer,
) error {
return postProcess(ctx, defaultProcessors, input, output)
}
var commitMessageProcessors = []processor{
fullIssuePatternProcessor,
comparePatternProcessor,
fullSha1PatternProcessor,
linkProcessor,
mentionProcessor,
issueIndexPatternProcessor,
commitCrossReferencePatternProcessor,
sha1CurrentPatternProcessor,
emailAddressProcessor,
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
emojiProcessor,
emojiShortCodeProcessor,
}
// RenderCommitMessage will use the same logic as PostProcess, but will disable
// the shortLinkProcessor and will add a defaultLinkProcessor if defaultLink is
// set, which changes every text node into a link to the passed default link.
func RenderCommitMessage(
ctx *RenderContext,
content string,
) (string, error) {
procs := commitMessageProcessors
if ctx.DefaultLink != "" {
// we don't have to fear data races, because being
// commitMessageProcessors of fixed len and cap, every time we append
// something to it the slice is realloc+copied, so append always
// generates the slice ex-novo.
procs = append(procs, genDefaultLinkProcessor(ctx.DefaultLink))
}
return renderProcessString(ctx, procs, content)
}
var commitMessageSubjectProcessors = []processor{
fullIssuePatternProcessor,
comparePatternProcessor,
fullSha1PatternProcessor,
linkProcessor,
mentionProcessor,
issueIndexPatternProcessor,
commitCrossReferencePatternProcessor,
sha1CurrentPatternProcessor,
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
emojiShortCodeProcessor,
emojiProcessor,
}
var emojiProcessors = []processor{
emojiShortCodeProcessor,
emojiProcessor,
}
// RenderCommitMessageSubject will use the same logic as PostProcess and
// RenderCommitMessage, but will disable the shortLinkProcessor and
// emailAddressProcessor, will add a defaultLinkProcessor if defaultLink is set,
// which changes every text node into a link to the passed default link.
func RenderCommitMessageSubject(
ctx *RenderContext,
content string,
) (string, error) {
procs := commitMessageSubjectProcessors
if ctx.DefaultLink != "" {
// we don't have to fear data races, because being
// commitMessageSubjectProcessors of fixed len and cap, every time we
// append something to it the slice is realloc+copied, so append always
// generates the slice ex-novo.
procs = append(procs, genDefaultLinkProcessor(ctx.DefaultLink))
}
return renderProcessString(ctx, procs, content)
}
// RenderIssueTitle to process title on individual issue/pull page
func RenderIssueTitle(
ctx *RenderContext,
title string,
) (string, error) {
return renderProcessString(ctx, []processor{
issueIndexPatternProcessor,
commitCrossReferencePatternProcessor,
sha1CurrentPatternProcessor,
emojiShortCodeProcessor,
emojiProcessor,
}, title)
}
func renderProcessString(ctx *RenderContext, procs []processor, content string) (string, error) {
var buf strings.Builder
if err := postProcess(ctx, procs, strings.NewReader(content), &buf); err != nil {
return "", err
}
return buf.String(), nil
}
// RenderDescriptionHTML will use similar logic as PostProcess, but will
// use a single special linkProcessor.
func RenderDescriptionHTML(
ctx *RenderContext,
content string,
) (string, error) {
return renderProcessString(ctx, []processor{
descriptionLinkProcessor,
emojiShortCodeProcessor,
emojiProcessor,
}, content)
}
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
// RenderEmoji for when we want to just process emoji and shortcodes
// in various places it isn't already run through the normal markdown processor
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
func RenderEmoji(
ctx *RenderContext,
content string,
) (string, error) {
return renderProcessString(ctx, emojiProcessors, content)
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
}
var (
tagCleaner = regexp.MustCompile(`<((?:/?\w+/\w+)|(?:/[\w ]+/)|(/?[hH][tT][mM][lL]\b)|(/?[hH][eE][aA][dD]\b))`)
nulCleaner = strings.NewReplacer("\000", "")
)
func postProcess(ctx *RenderContext, procs []processor, input io.Reader, output io.Writer) error {
defer ctx.Cancel()
// FIXME: don't read all content to memory
rawHTML, err := io.ReadAll(input)
if err != nil {
return err
}
// parse the HTML
node, err := html.Parse(io.MultiReader(
// prepend "<html><body>"
strings.NewReader("<html><body>"),
// Strip out nuls - they're always invalid
bytes.NewReader(tagCleaner.ReplaceAll([]byte(nulCleaner.Replace(string(rawHTML))), []byte("&lt;$1"))),
// close the tags
strings.NewReader("</body></html>"),
))
if err != nil {
return &postProcessError{"invalid HTML", err}
}
if node.Type == html.DocumentNode {
node = node.FirstChild
}
visitNode(ctx, procs, procs, node)
newNodes := make([]*html.Node, 0, 5)
if node.Data == "html" {
node = node.FirstChild
for node != nil && node.Data != "body" {
node = node.NextSibling
}
}
if node != nil {
if node.Data == "body" {
child := node.FirstChild
for child != nil {
newNodes = append(newNodes, child)
child = child.NextSibling
}
} else {
newNodes = append(newNodes, node)
}
}
// Render everything to buf.
for _, node := range newNodes {
2022-03-17 14:04:36 -04:00
if err := html.Render(output, node); err != nil {
return &postProcessError{"error rendering processed HTML", err}
}
}
return nil
}
func visitNode(ctx *RenderContext, procs, textProcs []processor, node *html.Node) {
// Add user-content- to IDs and "#" links if they don't already have them
for idx, attr := range node.Attr {
val := strings.TrimPrefix(attr.Val, "#")
notHasPrefix := !(strings.HasPrefix(val, "user-content-") || blackfridayExtRegex.MatchString(val))
if attr.Key == "id" && notHasPrefix {
node.Attr[idx].Val = "user-content-" + attr.Val
}
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
if attr.Key == "href" && strings.HasPrefix(attr.Val, "#") && notHasPrefix {
node.Attr[idx].Val = "#user-content-" + val
}
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
if attr.Key == "class" && attr.Val == "emoji" {
textProcs = nil
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
}
}
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
// We ignore code and pre.
switch node.Type {
case html.TextNode:
textNode(ctx, textProcs, node)
case html.ElementNode:
if node.Data == "img" {
for i, attr := range node.Attr {
if attr.Key != "src" {
continue
}
if len(attr.Val) > 0 && !isLinkStr(attr.Val) && !strings.HasPrefix(attr.Val, "data:image/") {
prefix := ctx.URLPrefix
if ctx.IsWiki {
prefix = util.URLJoin(prefix, "wiki", "raw")
}
prefix = strings.Replace(prefix, "/src/", "/media/", 1)
attr.Val = util.URLJoin(prefix, attr.Val)
}
attr.Val = camoHandleLink(attr.Val)
node.Attr[i] = attr
}
} else if node.Data == "a" {
// Restrict text in links to emojis
textProcs = emojiProcessors
} else if node.Data == "code" || node.Data == "pre" {
return
} else if node.Data == "i" {
for _, attr := range node.Attr {
if attr.Key != "class" {
continue
}
classes := strings.Split(attr.Val, " ")
for i, class := range classes {
if class == "icon" {
classes[0], classes[i] = classes[i], classes[0]
attr.Val = strings.Join(classes, " ")
// Remove all children of icons
child := node.FirstChild
for child != nil {
node.RemoveChild(child)
child = node.FirstChild
}
break
}
}
}
}
for n := node.FirstChild; n != nil; n = n.NextSibling {
visitNode(ctx, procs, textProcs, n)
}
}
// ignore everything else
}
// textNode runs the passed node through various processors, in order to handle
// all kinds of special links handled by the post-processing.
func textNode(ctx *RenderContext, procs []processor, node *html.Node) {
for _, processor := range procs {
processor(ctx, node)
}
}
// createKeyword() renders a highlighted version of an action keyword
func createKeyword(content string) *html.Node {
span := &html.Node{
Type: html.ElementNode,
Data: atom.Span.String(),
Attr: []html.Attribute{},
}
span.Attr = append(span.Attr, html.Attribute{Key: "class", Val: keywordClass})
text := &html.Node{
Type: html.TextNode,
Data: content,
}
span.AppendChild(text)
return span
}
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
func createEmoji(content, class, name string) *html.Node {
span := &html.Node{
Type: html.ElementNode,
Data: atom.Span.String(),
Attr: []html.Attribute{},
}
if class != "" {
span.Attr = append(span.Attr, html.Attribute{Key: "class", Val: class})
}
if name != "" {
span.Attr = append(span.Attr, html.Attribute{Key: "aria-label", Val: name})
}
text := &html.Node{
Type: html.TextNode,
Data: content,
}
span.AppendChild(text)
return span
}
2021-06-29 10:28:38 -04:00
func createCustomEmoji(alias string) *html.Node {
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
span := &html.Node{
Type: html.ElementNode,
Data: atom.Span.String(),
Attr: []html.Attribute{},
}
2021-06-29 10:28:38 -04:00
span.Attr = append(span.Attr, html.Attribute{Key: "class", Val: "emoji"})
span.Attr = append(span.Attr, html.Attribute{Key: "aria-label", Val: alias})
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
img := &html.Node{
Type: html.ElementNode,
DataAtom: atom.Img,
Data: "img",
Attr: []html.Attribute{},
}
2021-06-29 10:28:38 -04:00
img.Attr = append(img.Attr, html.Attribute{Key: "alt", Val: ":" + alias + ":"})
img.Attr = append(img.Attr, html.Attribute{Key: "src", Val: setting.StaticURLPrefix + "/assets/img/emoji/" + alias + ".png"})
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
span.AppendChild(img)
return span
}
func createLink(href, content, class string) *html.Node {
a := &html.Node{
Type: html.ElementNode,
Data: atom.A.String(),
Attr: []html.Attribute{{Key: "href", Val: href}},
}
if class != "" {
a.Attr = append(a.Attr, html.Attribute{Key: "class", Val: class})
}
text := &html.Node{
Type: html.TextNode,
Data: content,
}
a.AppendChild(text)
return a
}
func createCodeLink(href, content, class string) *html.Node {
a := &html.Node{
Type: html.ElementNode,
Data: atom.A.String(),
Attr: []html.Attribute{{Key: "href", Val: href}},
}
if class != "" {
a.Attr = append(a.Attr, html.Attribute{Key: "class", Val: class})
}
text := &html.Node{
Type: html.TextNode,
Data: content,
}
code := &html.Node{
Type: html.ElementNode,
Data: atom.Code.String(),
Attr: []html.Attribute{{Key: "class", Val: "nohighlight"}},
}
code.AppendChild(text)
a.AppendChild(code)
return a
}
// replaceContent takes text node, and in its content it replaces a section of
// it with the specified newNode.
func replaceContent(node *html.Node, i, j int, newNode *html.Node) {
replaceContentList(node, i, j, []*html.Node{newNode})
}
// replaceContentList takes text node, and in its content it replaces a section of
// it with the specified newNodes. An example to visualize how this can work can
// be found here: https://play.golang.org/p/5zP8NnHZ03s
func replaceContentList(node *html.Node, i, j int, newNodes []*html.Node) {
// get the data before and after the match
before := node.Data[:i]
after := node.Data[j:]
// Replace in the current node the text, so that it is only what it is
// supposed to have.
node.Data = before
// Get the current next sibling, before which we place the replaced data,
// and after that we place the new text node.
nextSibling := node.NextSibling
for _, n := range newNodes {
node.Parent.InsertBefore(n, nextSibling)
}
if after != "" {
node.Parent.InsertBefore(&html.Node{
Type: html.TextNode,
Data: after,
}, nextSibling)
}
}
func mentionProcessor(ctx *RenderContext, node *html.Node) {
start := 0
next := node.NextSibling
for node != nil && node != next && start < len(node.Data) {
// We replace only the first mention; other mentions will be addressed later
found, loc := references.FindFirstMentionBytes([]byte(node.Data[start:]))
if !found {
return
}
loc.Start += start
loc.End += start
mention := node.Data[loc.Start:loc.End]
var teams string
teams, ok := ctx.Metas["teams"]
// FIXME: util.URLJoin may not be necessary here:
// - setting.AppURL is defined to have a terminal '/' so unless mention[1:]
// is an AppSubURL link we can probably fallback to concatenation.
// team mention should follow @orgName/teamName style
if ok && strings.Contains(mention, "/") {
mentionOrgAndTeam := strings.Split(mention, "/")
if mentionOrgAndTeam[0][1:] == ctx.Metas["org"] && strings.Contains(teams, ","+strings.ToLower(mentionOrgAndTeam[1])+",") {
replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(setting.AppURL, "org", ctx.Metas["org"], "teams", mentionOrgAndTeam[1]), mention, "mention"))
node = node.NextSibling.NextSibling
start = 0
continue
}
start = loc.End
continue
}
mentionedUsername := mention[1:]
if DefaultProcessorHelper.IsUsernameMentionable != nil && DefaultProcessorHelper.IsUsernameMentionable(ctx.Ctx, mentionedUsername) {
replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(setting.AppURL, mentionedUsername), mention, "mention"))
node = node.NextSibling.NextSibling
} else {
node = node.NextSibling
}
start = 0
}
}
func shortLinkProcessor(ctx *RenderContext, node *html.Node) {
shortLinkProcessorFull(ctx, node, false)
}
func shortLinkProcessorFull(ctx *RenderContext, node *html.Node, noLink bool) {
next := node.NextSibling
for node != nil && node != next {
m := shortLinkPattern.FindStringSubmatchIndex(node.Data)
if m == nil {
return
}
content := node.Data[m[2]:m[3]]
tail := node.Data[m[4]:m[5]]
props := make(map[string]string)
// MediaWiki uses [[link|text]], while GitHub uses [[text|link]]
// It makes page handling terrible, but we prefer GitHub syntax
// And fall back to MediaWiki only when it is obvious from the look
// Of text and link contents
sl := strings.Split(content, "|")
for _, v := range sl {
if equalPos := strings.IndexByte(v, '='); equalPos == -1 {
// There is no equal in this argument; this is a mandatory arg
if props["name"] == "" {
if isLinkStr(v) {
// If we clearly see it is a link, we save it so
// But first we need to ensure, that if both mandatory args provided
// look like links, we stick to GitHub syntax
if props["link"] != "" {
props["name"] = props["link"]
}
props["link"] = strings.TrimSpace(v)
} else {
props["name"] = v
}
} else {
props["link"] = strings.TrimSpace(v)
}
} else {
// There is an equal; optional argument.
sep := strings.IndexByte(v, '=')
key, val := v[:sep], html.UnescapeString(v[sep+1:])
// When parsing HTML, x/net/html will change all quotes which are
// not used for syntax into UTF-8 quotes. So checking val[0] won't
// be enough, since that only checks a single byte.
if len(val) > 1 {
if (strings.HasPrefix(val, "“") && strings.HasSuffix(val, "”")) ||
(strings.HasPrefix(val, "") && strings.HasSuffix(val, "")) {
const lenQuote = len("")
val = val[lenQuote : len(val)-lenQuote]
} else if (strings.HasPrefix(val, "\"") && strings.HasSuffix(val, "\"")) ||
(strings.HasPrefix(val, "'") && strings.HasSuffix(val, "'")) {
val = val[1 : len(val)-1]
} else if strings.HasPrefix(val, "'") && strings.HasSuffix(val, "") {
const lenQuote = len("")
val = val[1 : len(val)-lenQuote]
}
}
props[key] = val
}
}
var name, link string
if props["link"] != "" {
link = props["link"]
} else if props["name"] != "" {
link = props["name"]
}
if props["title"] != "" {
name = props["title"]
} else if props["name"] != "" {
name = props["name"]
} else {
name = link
}
name += tail
image := false
switch ext := filepath.Ext(link); ext {
// fast path: empty string, ignore
case "":
// leave image as false
case ".jpg", ".jpeg", ".png", ".tif", ".tiff", ".webp", ".gif", ".bmp", ".ico", ".svg":
image = true
2018-06-15 08:42:49 -04:00
}
childNode := &html.Node{}
linkNode := &html.Node{
FirstChild: childNode,
LastChild: childNode,
Type: html.ElementNode,
Data: "a",
DataAtom: atom.A,
}
childNode.Parent = linkNode
absoluteLink := isLinkStr(link)
if !absoluteLink {
if image {
link = strings.ReplaceAll(link, " ", "+")
} else {
link = strings.ReplaceAll(link, " ", "-")
}
if !strings.Contains(link, "/") {
link = url.PathEscape(link)
}
}
urlPrefix := ctx.URLPrefix
if image {
if !absoluteLink {
if IsSameDomain(urlPrefix) {
urlPrefix = strings.Replace(urlPrefix, "/src/", "/raw/", 1)
}
if ctx.IsWiki {
link = util.URLJoin("wiki", "raw", link)
}
link = util.URLJoin(urlPrefix, link)
}
title := props["title"]
if title == "" {
title = props["alt"]
}
if title == "" {
title = path.Base(name)
}
alt := props["alt"]
if alt == "" {
alt = name
}
// make the childNode an image - if we can, we also place the alt
childNode.Type = html.ElementNode
childNode.Data = "img"
childNode.DataAtom = atom.Img
childNode.Attr = []html.Attribute{
{Key: "src", Val: link},
{Key: "title", Val: title},
{Key: "alt", Val: alt},
}
if alt == "" {
childNode.Attr = childNode.Attr[:2]
}
} else {
if !absoluteLink {
if ctx.IsWiki {
link = util.URLJoin("wiki", link)
}
link = util.URLJoin(urlPrefix, link)
}
childNode.Type = html.TextNode
childNode.Data = name
}
if noLink {
linkNode = childNode
} else {
linkNode.Attr = []html.Attribute{{Key: "href", Val: link}}
}
replaceContent(node, m[0], m[1], linkNode)
node = node.NextSibling.NextSibling
}
}
func fullIssuePatternProcessor(ctx *RenderContext, node *html.Node) {
if ctx.Metas == nil {
return
}
next := node.NextSibling
for node != nil && node != next {
m := getIssueFullPattern().FindStringSubmatchIndex(node.Data)
if m == nil {
return
}
Append `(comment)` when a link points at a comment rather than the whole issue (#23734) Close #23671 For the feature mentioned above, this PR append ' (comment)' to the rendered html if it is a hashcomment. After the PR, type in the following ``` pull request from other repo: http://localhost:3000/testOrg/testOrgRepo/pulls/2 pull request from this repo: http://localhost:3000/aaa/testA/pulls/2 issue comment from this repo: http://localhost:3000/aaa/testA/issues/1#issuecomment-18 http://localhost:3000/aaa/testA/pulls/2#issue-9 issue comment from other repo: http://localhost:3000/testOrg/testOrgRepo/pulls/2#issuecomment-24 http://localhost:3000/testOrg/testOrgRepo/pulls/2#issue ``` Gives: <img width="687" alt="截屏2023-03-27 13 53 06" src="https://user-images.githubusercontent.com/17645053/227852387-2b218e0d-3468-4d90-ad81-d702ddd17fd2.png"> Other than the above feature, this PR also includes two other changes: 1 Right now, the render of links from file changed tab in pull request might not be very proper, for example, if type in the following. (not sure if this is an issue or design, if not an issue, I will revert the changes). example on [try.gitea.io](https://try.gitea.io/HesterG/testrepo/pulls/1) ``` https://try.gitea.io/HesterG/testrepo/pulls/1/files#issuecomment-162725 https://try.gitea.io/HesterG/testrepo/pulls/1/files ``` it will render the following <img width="899" alt="截屏2023-03-24 15 41 37" src="https://user-images.githubusercontent.com/17645053/227456117-5eccedb7-9118-4540-929d-aee9a76de852.png"> In this PR, skip processing the link into a ref issue if it is a link from files changed tab in pull request After: type in following ``` hash comment on files changed tab: http://localhost:3000/testOrg/testOrgRepo/pulls/2/files#issuecomment-24 files changed link: http://localhost:3000/testOrg/testOrgRepo/pulls/2/files ``` Gives <img width="708" alt="截屏2023-03-27 22 09 02" src="https://user-images.githubusercontent.com/17645053/227964273-5dc06c50-3713-489c-b05d-d95367d0ab0f.png"> 2 Right now, after editing the comment area, there will not be tippys attached to `ref-issue`; and no tippy attached on preview as well. example: https://user-images.githubusercontent.com/17645053/227850540-5ae34e2d-b1d7-4d0d-9726-7701bf825d1f.mov In this PR, in frontend, make sure tippy is added after editing the comment, and to the comment on preview tab After: https://user-images.githubusercontent.com/17645053/227853777-06f56b4c-1148-467c-b6f7-f79418e67504.mov
2023-04-03 04:02:57 -04:00
mDiffView := getFilesChangedFullPattern().FindStringSubmatchIndex(node.Data)
// leave it as it is if the link is from "Files Changed" tab in PR Diff View https://domain/org/repo/pulls/27/files
if mDiffView != nil {
return
}
link := node.Data[m[0]:m[1]]
Append `(comment)` when a link points at a comment rather than the whole issue (#23734) Close #23671 For the feature mentioned above, this PR append ' (comment)' to the rendered html if it is a hashcomment. After the PR, type in the following ``` pull request from other repo: http://localhost:3000/testOrg/testOrgRepo/pulls/2 pull request from this repo: http://localhost:3000/aaa/testA/pulls/2 issue comment from this repo: http://localhost:3000/aaa/testA/issues/1#issuecomment-18 http://localhost:3000/aaa/testA/pulls/2#issue-9 issue comment from other repo: http://localhost:3000/testOrg/testOrgRepo/pulls/2#issuecomment-24 http://localhost:3000/testOrg/testOrgRepo/pulls/2#issue ``` Gives: <img width="687" alt="截屏2023-03-27 13 53 06" src="https://user-images.githubusercontent.com/17645053/227852387-2b218e0d-3468-4d90-ad81-d702ddd17fd2.png"> Other than the above feature, this PR also includes two other changes: 1 Right now, the render of links from file changed tab in pull request might not be very proper, for example, if type in the following. (not sure if this is an issue or design, if not an issue, I will revert the changes). example on [try.gitea.io](https://try.gitea.io/HesterG/testrepo/pulls/1) ``` https://try.gitea.io/HesterG/testrepo/pulls/1/files#issuecomment-162725 https://try.gitea.io/HesterG/testrepo/pulls/1/files ``` it will render the following <img width="899" alt="截屏2023-03-24 15 41 37" src="https://user-images.githubusercontent.com/17645053/227456117-5eccedb7-9118-4540-929d-aee9a76de852.png"> In this PR, skip processing the link into a ref issue if it is a link from files changed tab in pull request After: type in following ``` hash comment on files changed tab: http://localhost:3000/testOrg/testOrgRepo/pulls/2/files#issuecomment-24 files changed link: http://localhost:3000/testOrg/testOrgRepo/pulls/2/files ``` Gives <img width="708" alt="截屏2023-03-27 22 09 02" src="https://user-images.githubusercontent.com/17645053/227964273-5dc06c50-3713-489c-b05d-d95367d0ab0f.png"> 2 Right now, after editing the comment area, there will not be tippys attached to `ref-issue`; and no tippy attached on preview as well. example: https://user-images.githubusercontent.com/17645053/227850540-5ae34e2d-b1d7-4d0d-9726-7701bf825d1f.mov In this PR, in frontend, make sure tippy is added after editing the comment, and to the comment on preview tab After: https://user-images.githubusercontent.com/17645053/227853777-06f56b4c-1148-467c-b6f7-f79418e67504.mov
2023-04-03 04:02:57 -04:00
text := "#" + node.Data[m[2]:m[3]]
// if m[4] and m[5] is not -1, then link is to a comment
// indicate that in the text by appending (comment)
if m[4] != -1 && m[5] != -1 {
if locale, ok := ctx.Ctx.Value(translation.ContextKey).(translation.Locale); ok {
text += " " + locale.Tr("repo.from_comment")
} else {
text += " (comment)"
}
}
// extract repo and org name from matched link like
// http://localhost:3000/gituser/myrepo/issues/1
linkParts := strings.Split(link, "/")
matchOrg := linkParts[len(linkParts)-4]
matchRepo := linkParts[len(linkParts)-3]
if matchOrg == ctx.Metas["user"] && matchRepo == ctx.Metas["repo"] {
Append `(comment)` when a link points at a comment rather than the whole issue (#23734) Close #23671 For the feature mentioned above, this PR append ' (comment)' to the rendered html if it is a hashcomment. After the PR, type in the following ``` pull request from other repo: http://localhost:3000/testOrg/testOrgRepo/pulls/2 pull request from this repo: http://localhost:3000/aaa/testA/pulls/2 issue comment from this repo: http://localhost:3000/aaa/testA/issues/1#issuecomment-18 http://localhost:3000/aaa/testA/pulls/2#issue-9 issue comment from other repo: http://localhost:3000/testOrg/testOrgRepo/pulls/2#issuecomment-24 http://localhost:3000/testOrg/testOrgRepo/pulls/2#issue ``` Gives: <img width="687" alt="截屏2023-03-27 13 53 06" src="https://user-images.githubusercontent.com/17645053/227852387-2b218e0d-3468-4d90-ad81-d702ddd17fd2.png"> Other than the above feature, this PR also includes two other changes: 1 Right now, the render of links from file changed tab in pull request might not be very proper, for example, if type in the following. (not sure if this is an issue or design, if not an issue, I will revert the changes). example on [try.gitea.io](https://try.gitea.io/HesterG/testrepo/pulls/1) ``` https://try.gitea.io/HesterG/testrepo/pulls/1/files#issuecomment-162725 https://try.gitea.io/HesterG/testrepo/pulls/1/files ``` it will render the following <img width="899" alt="截屏2023-03-24 15 41 37" src="https://user-images.githubusercontent.com/17645053/227456117-5eccedb7-9118-4540-929d-aee9a76de852.png"> In this PR, skip processing the link into a ref issue if it is a link from files changed tab in pull request After: type in following ``` hash comment on files changed tab: http://localhost:3000/testOrg/testOrgRepo/pulls/2/files#issuecomment-24 files changed link: http://localhost:3000/testOrg/testOrgRepo/pulls/2/files ``` Gives <img width="708" alt="截屏2023-03-27 22 09 02" src="https://user-images.githubusercontent.com/17645053/227964273-5dc06c50-3713-489c-b05d-d95367d0ab0f.png"> 2 Right now, after editing the comment area, there will not be tippys attached to `ref-issue`; and no tippy attached on preview as well. example: https://user-images.githubusercontent.com/17645053/227850540-5ae34e2d-b1d7-4d0d-9726-7701bf825d1f.mov In this PR, in frontend, make sure tippy is added after editing the comment, and to the comment on preview tab After: https://user-images.githubusercontent.com/17645053/227853777-06f56b4c-1148-467c-b6f7-f79418e67504.mov
2023-04-03 04:02:57 -04:00
replaceContent(node, m[0], m[1], createLink(link, text, "ref-issue"))
} else {
Append `(comment)` when a link points at a comment rather than the whole issue (#23734) Close #23671 For the feature mentioned above, this PR append ' (comment)' to the rendered html if it is a hashcomment. After the PR, type in the following ``` pull request from other repo: http://localhost:3000/testOrg/testOrgRepo/pulls/2 pull request from this repo: http://localhost:3000/aaa/testA/pulls/2 issue comment from this repo: http://localhost:3000/aaa/testA/issues/1#issuecomment-18 http://localhost:3000/aaa/testA/pulls/2#issue-9 issue comment from other repo: http://localhost:3000/testOrg/testOrgRepo/pulls/2#issuecomment-24 http://localhost:3000/testOrg/testOrgRepo/pulls/2#issue ``` Gives: <img width="687" alt="截屏2023-03-27 13 53 06" src="https://user-images.githubusercontent.com/17645053/227852387-2b218e0d-3468-4d90-ad81-d702ddd17fd2.png"> Other than the above feature, this PR also includes two other changes: 1 Right now, the render of links from file changed tab in pull request might not be very proper, for example, if type in the following. (not sure if this is an issue or design, if not an issue, I will revert the changes). example on [try.gitea.io](https://try.gitea.io/HesterG/testrepo/pulls/1) ``` https://try.gitea.io/HesterG/testrepo/pulls/1/files#issuecomment-162725 https://try.gitea.io/HesterG/testrepo/pulls/1/files ``` it will render the following <img width="899" alt="截屏2023-03-24 15 41 37" src="https://user-images.githubusercontent.com/17645053/227456117-5eccedb7-9118-4540-929d-aee9a76de852.png"> In this PR, skip processing the link into a ref issue if it is a link from files changed tab in pull request After: type in following ``` hash comment on files changed tab: http://localhost:3000/testOrg/testOrgRepo/pulls/2/files#issuecomment-24 files changed link: http://localhost:3000/testOrg/testOrgRepo/pulls/2/files ``` Gives <img width="708" alt="截屏2023-03-27 22 09 02" src="https://user-images.githubusercontent.com/17645053/227964273-5dc06c50-3713-489c-b05d-d95367d0ab0f.png"> 2 Right now, after editing the comment area, there will not be tippys attached to `ref-issue`; and no tippy attached on preview as well. example: https://user-images.githubusercontent.com/17645053/227850540-5ae34e2d-b1d7-4d0d-9726-7701bf825d1f.mov In this PR, in frontend, make sure tippy is added after editing the comment, and to the comment on preview tab After: https://user-images.githubusercontent.com/17645053/227853777-06f56b4c-1148-467c-b6f7-f79418e67504.mov
2023-04-03 04:02:57 -04:00
text = matchOrg + "/" + matchRepo + text
replaceContent(node, m[0], m[1], createLink(link, text, "ref-issue"))
}
node = node.NextSibling.NextSibling
}
}
func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) {
if ctx.Metas == nil {
return
}
var (
found bool
ref *references.RenderizableReference
)
next := node.NextSibling
for node != nil && node != next {
_, hasExtTrackFormat := ctx.Metas["format"]
// Repos with external issue trackers might still need to reference local PRs
// We need to concern with the first one that shows up in the text, whichever it is
isNumericStyle := ctx.Metas["style"] == "" || ctx.Metas["style"] == IssueNameStyleNumeric
foundNumeric, refNumeric := references.FindRenderizableReferenceNumeric(node.Data, hasExtTrackFormat && !isNumericStyle)
switch ctx.Metas["style"] {
case "", IssueNameStyleNumeric:
found, ref = foundNumeric, refNumeric
case IssueNameStyleAlphanumeric:
found, ref = references.FindRenderizableReferenceAlphanumeric(node.Data)
case IssueNameStyleRegexp:
pattern, err := regexplru.GetCompiled(ctx.Metas["regexp"])
if err != nil {
return
}
found, ref = references.FindRenderizableReferenceRegexp(node.Data, pattern)
}
// Repos with external issue trackers might still need to reference local PRs
// We need to concern with the first one that shows up in the text, whichever it is
if hasExtTrackFormat && !isNumericStyle && refNumeric != nil {
// If numeric (PR) was found, and it was BEFORE the non-numeric pattern, use that
// Allow a free-pass when non-numeric pattern wasn't found.
if found && (ref == nil || refNumeric.RefLocation.Start < ref.RefLocation.Start) {
found = foundNumeric
ref = refNumeric
}
}
if !found {
return
}
var link *html.Node
reftext := node.Data[ref.RefLocation.Start:ref.RefLocation.End]
if hasExtTrackFormat && !ref.IsPull {
ctx.Metas["index"] = ref.Issue
res, err := vars.Expand(ctx.Metas["format"], ctx.Metas)
if err != nil {
// here we could just log the error and continue the rendering
log.Error("unable to expand template vars for ref %s, err: %v", ref.Issue, err)
}
link = createLink(res, reftext, "ref-issue ref-external-issue")
} else {
// Path determines the type of link that will be rendered. It's unknown at this point whether
// the linked item is actually a PR or an issue. Luckily it's of no real consequence because
// Gitea will redirect on click as appropriate.
path := "issues"
if ref.IsPull {
path = "pulls"
}
if ref.Owner == "" {
link = createLink(util.URLJoin(setting.AppURL, ctx.Metas["user"], ctx.Metas["repo"], path, ref.Issue), reftext, "ref-issue")
} else {
link = createLink(util.URLJoin(setting.AppURL, ref.Owner, ref.Name, path, ref.Issue), reftext, "ref-issue")
}
}
if ref.Action == references.XRefActionNone {
replaceContent(node, ref.RefLocation.Start, ref.RefLocation.End, link)
node = node.NextSibling.NextSibling
continue
}
// Decorate action keywords if actionable
var keyword *html.Node
if references.IsXrefActionable(ref, hasExtTrackFormat) {
keyword = createKeyword(node.Data[ref.ActionLocation.Start:ref.ActionLocation.End])
} else {
keyword = &html.Node{
Type: html.TextNode,
Data: node.Data[ref.ActionLocation.Start:ref.ActionLocation.End],
}
}
spaces := &html.Node{
Type: html.TextNode,
Data: node.Data[ref.ActionLocation.End:ref.RefLocation.Start],
}
replaceContentList(node, ref.ActionLocation.Start, ref.RefLocation.End, []*html.Node{keyword, spaces, link})
node = node.NextSibling.NextSibling.NextSibling.NextSibling
}
}
func commitCrossReferencePatternProcessor(ctx *RenderContext, node *html.Node) {
next := node.NextSibling
for node != nil && node != next {
found, ref := references.FindRenderizableCommitCrossReference(node.Data)
if !found {
return
}
reftext := ref.Owner + "/" + ref.Name + "@" + base.ShortSha(ref.CommitSha)
link := createLink(util.URLJoin(setting.AppSubURL, ref.Owner, ref.Name, "commit", ref.CommitSha), reftext, "commit")
replaceContent(node, ref.RefLocation.Start, ref.RefLocation.End, link)
node = node.NextSibling.NextSibling
}
}
// fullSha1PatternProcessor renders SHA containing URLs
func fullSha1PatternProcessor(ctx *RenderContext, node *html.Node) {
if ctx.Metas == nil {
return
}
next := node.NextSibling
for node != nil && node != next {
m := anySHA1Pattern.FindStringSubmatchIndex(node.Data)
if m == nil {
return
}
urlFull := node.Data[m[0]:m[1]]
text := base.ShortSha(node.Data[m[2]:m[3]])
// 3rd capture group matches a optional path
subpath := ""
if m[5] > 0 {
subpath = node.Data[m[4]:m[5]]
}
// 4th capture group matches a optional url hash
hash := ""
if m[7] > 0 {
hash = node.Data[m[6]:m[7]][1:]
}
start := m[0]
end := m[1]
// If url ends in '.', it's very likely that it is not part of the
// actual url but used to finish a sentence.
if strings.HasSuffix(urlFull, ".") {
end--
urlFull = urlFull[:len(urlFull)-1]
if hash != "" {
hash = hash[:len(hash)-1]
} else if subpath != "" {
subpath = subpath[:len(subpath)-1]
}
}
if subpath != "" {
text += subpath
}
if hash != "" {
text += " (" + hash + ")"
}
replaceContent(node, start, end, createCodeLink(urlFull, text, "commit"))
node = node.NextSibling.NextSibling
}
}
func comparePatternProcessor(ctx *RenderContext, node *html.Node) {
if ctx.Metas == nil {
return
}
next := node.NextSibling
for node != nil && node != next {
m := comparePattern.FindStringSubmatchIndex(node.Data)
if m == nil {
return
}
// Ensure that every group (m[0]...m[7]) has a match
for i := 0; i < 8; i++ {
if m[i] == -1 {
return
}
}
urlFull := node.Data[m[0]:m[1]]
text1 := base.ShortSha(node.Data[m[2]:m[3]])
textDots := base.ShortSha(node.Data[m[4]:m[5]])
text2 := base.ShortSha(node.Data[m[6]:m[7]])
hash := ""
if m[9] > 0 {
hash = node.Data[m[8]:m[9]][1:]
}
start := m[0]
end := m[1]
// If url ends in '.', it's very likely that it is not part of the
// actual url but used to finish a sentence.
if strings.HasSuffix(urlFull, ".") {
end--
urlFull = urlFull[:len(urlFull)-1]
if hash != "" {
hash = hash[:len(hash)-1]
} else if text2 != "" {
text2 = text2[:len(text2)-1]
}
}
text := text1 + textDots + text2
if hash != "" {
text += " (" + hash + ")"
}
replaceContent(node, start, end, createCodeLink(urlFull, text, "compare"))
node = node.NextSibling.NextSibling
}
}
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
// emojiShortCodeProcessor for rendering text like :smile: into emoji
func emojiShortCodeProcessor(ctx *RenderContext, node *html.Node) {
start := 0
next := node.NextSibling
for node != nil && node != next && start < len(node.Data) {
m := EmojiShortCodeRegex.FindStringSubmatchIndex(node.Data[start:])
if m == nil {
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
return
}
m[0] += start
m[1] += start
start = m[1]
alias := node.Data[m[0]:m[1]]
alias = strings.ReplaceAll(alias, ":", "")
converted := emoji.FromAlias(alias)
if converted == nil {
// check if this is a custom reaction
2021-06-29 10:28:38 -04:00
if _, exist := setting.UI.CustomEmojisMap[alias]; exist {
replaceContent(node, m[0], m[1], createCustomEmoji(alias))
node = node.NextSibling.NextSibling
start = 0
continue
}
continue
}
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
replaceContent(node, m[0], m[1], createEmoji(converted.Emoji, "emoji", converted.Description))
node = node.NextSibling.NextSibling
start = 0
}
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
}
// emoji processor to match emoji and add emoji class
func emojiProcessor(ctx *RenderContext, node *html.Node) {
start := 0
next := node.NextSibling
for node != nil && node != next && start < len(node.Data) {
m := emoji.FindEmojiSubmatchIndex(node.Data[start:])
if m == nil {
return
}
m[0] += start
m[1] += start
codepoint := node.Data[m[0]:m[1]]
start = m[1]
val := emoji.FromCode(codepoint)
if val != nil {
replaceContent(node, m[0], m[1], createEmoji(codepoint, "emoji", val.Description))
node = node.NextSibling.NextSibling
start = 0
}
Support unicode emojis and remove emojify.js (#11032) * Support unicode emojis and remove emojify.js This PR replaces all use of emojify.js and adds unicode emoji support to various areas of gitea. This works in a few ways: First it adds emoji parsing support into gitea itself. This allows us to * Render emojis from valid alias (:smile:) * Detect unicode emojis and let us put them in their own class with proper aria-labels and styling * Easily allow for custom "emoji" * Support all emoji rendering and features without javascript * Uses plain unicode and lets the system render in appropriate emoji font * Doesn't leave us relying on external sources for updates/fixes/features That same list of emoji is also used to create a json file which replaces the part of emojify.js that populates the emoji search tribute. This file is about 35KB with GZIP turned on and I've set it to load after the page renders to not hinder page load time (and this removes loading emojify.js also) For custom "emoji" it uses a pretty simple scheme of just looking for /emojis/img/name.png where name is something a user has put in the "allowed reactions" setting we already have. The gitea reaction that was previously hard coded into a forked copy of emojify.js is included and works as a custom reaction under this method. The emoji data sourced here is from https://github.com/github/gemoji which is the gem library Github uses for their emoji rendering (and a data source for other sites). So we should be able to easily render any emoji and :alias: that Github can, removing any errors from migrated content. They also update it as well, so we can sync when there are new unicode emoji lists released. I've included a slimmed down and slightly modified forked copy of https://github.com/knq/emoji to make up our own emoji module. The code is pretty straight forward and again allows us to have a lot of flexibility in what happens. I had seen a few comments about performance in some of the other threads if we render this ourselves, but there doesn't seem to be any issue here. In a test it can parse, convert, and render 1,000 emojis inside of a large markdown table in about 100ms on my laptop (which is many more emojis than will ever be in any normal issue). This also prevents any flickering and other weirdness from using javascript to render some things while using go for others. Not included here are image fall back URLS. I don't really think they are necessary for anything new being written in 2020. However, managing the emoji ourselves would allow us to add these as a feature later on if it seems necessary. Fixes: https://github.com/go-gitea/gitea/issues/9182 Fixes: https://github.com/go-gitea/gitea/issues/8974 Fixes: https://github.com/go-gitea/gitea/issues/8953 Fixes: https://github.com/go-gitea/gitea/issues/6628 Fixes: https://github.com/go-gitea/gitea/issues/5130 * add new shared function emojiHTML * don't increase emoji size in issue title * Update templates/repo/issue/view_content/add_reaction.tmpl Co-Authored-By: 6543 <6543@obermui.de> * Support for emoji rendering in various templates * Render code and review comments as they should be * Better way to handle mail subjects * insert unicode from tribute selection * Add template helper for plain text when needed * Use existing replace function I forgot about * Don't include emoji greater than Unicode Version 12 Only include emoji and aliases in JSON * Update build/generate-emoji.go * Tweak regex slightly to really match everything including random invisible characters. Run tests for every emoji we have * final updates * code review * code review * hard code gitea custom emoji to match previous behavior * Update .eslintrc Co-Authored-By: silverwind <me@silverwind.io> * disable preempt Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
2020-04-28 14:05:39 -04:00
}
}
// sha1CurrentPatternProcessor renders SHA1 strings to corresponding links that
// are assumed to be in the same repository.
func sha1CurrentPatternProcessor(ctx *RenderContext, node *html.Node) {
if ctx.Metas == nil || ctx.Metas["user"] == "" || ctx.Metas["repo"] == "" || ctx.Metas["repoPath"] == "" {
Check commit message hashes before making links (#7713) * Check commit message hashes before making links Previously, when formatting commit messages, anything that looked like SHA1 hashes was turned into a link using regex. This meant that certain phrases or numbers such as `777777` or `deadbeef` could be recognized as a commit even if the repository has no commit with those hashes. This change will make it so that anything that looks like a SHA1 hash using regex will then also be checked to ensure that there is a commit in the repository with that hash before making a link. Signed-off-by: Gary Kim <gary@garykim.dev> * Use gogit to check if commit exists This commit modifies the commit hash check in the render for commit messages to use gogit for better performance. Signed-off-by: Gary Kim <gary@garykim.dev> * Make code cleaner Signed-off-by: Gary Kim <gary@garykim.dev> * Use rev-parse to check if commit exists Signed-off-by: Gary Kim <gary@garykim.dev> * Add and modify tests for checking hashes in html link rendering Signed-off-by: Gary Kim <gary@garykim.dev> * Return error in sha1CurrentPatternProcessor Co-Authored-By: mrsdizzie <info@mrsdizzie.com> * Import Gitea log module Signed-off-by: Gary Kim <gary@garykim.dev> * Revert "Return error in sha1CurrentPatternProcessor" This reverts commit 28f561cac46ef7e51aa26aefcbe9aca4671366a6. Signed-off-by: Gary Kim <gary@garykim.dev> * Add debug logging to sha1CurrentPatternProcessor This will log errors by the git command run in sha1CurrentPatternProcessor if the error is one that was unexpected. Signed-off-by: Gary Kim <gary@garykim.dev>
2019-08-14 04:04:55 -04:00
return
}
start := 0
next := node.NextSibling
if ctx.ShaExistCache == nil {
ctx.ShaExistCache = make(map[string]bool)
}
for node != nil && node != next && start < len(node.Data) {
m := sha1CurrentPattern.FindStringSubmatchIndex(node.Data[start:])
if m == nil {
return
}
m[2] += start
m[3] += start
hash := node.Data[m[2]:m[3]]
// The regex does not lie, it matches the hash pattern.
// However, a regex cannot know if a hash actually exists or not.
// We could assume that a SHA1 hash should probably contain alphas AND numerics
// but that is not always the case.
// Although unlikely, deadbeef and 1234567 are valid short forms of SHA1 hash
// as used by git and github for linking and thus we have to do similar.
// Because of this, we check to make sure that a matched hash is actually
// a commit in the repository before making it a link.
// check cache first
exist, inCache := ctx.ShaExistCache[hash]
if !inCache {
if ctx.GitRepo == nil {
var err error
ctx.GitRepo, err = git.OpenRepository(ctx.Ctx, ctx.Metas["repoPath"])
if err != nil {
log.Error("unable to open repository: %s Error: %v", ctx.Metas["repoPath"], err)
return
}
ctx.AddCancel(func() {
ctx.GitRepo.Close()
ctx.GitRepo = nil
})
}
exist = ctx.GitRepo.IsObjectExist(hash)
ctx.ShaExistCache[hash] = exist
}
if !exist {
start = m[3]
continue
Check commit message hashes before making links (#7713) * Check commit message hashes before making links Previously, when formatting commit messages, anything that looked like SHA1 hashes was turned into a link using regex. This meant that certain phrases or numbers such as `777777` or `deadbeef` could be recognized as a commit even if the repository has no commit with those hashes. This change will make it so that anything that looks like a SHA1 hash using regex will then also be checked to ensure that there is a commit in the repository with that hash before making a link. Signed-off-by: Gary Kim <gary@garykim.dev> * Use gogit to check if commit exists This commit modifies the commit hash check in the render for commit messages to use gogit for better performance. Signed-off-by: Gary Kim <gary@garykim.dev> * Make code cleaner Signed-off-by: Gary Kim <gary@garykim.dev> * Use rev-parse to check if commit exists Signed-off-by: Gary Kim <gary@garykim.dev> * Add and modify tests for checking hashes in html link rendering Signed-off-by: Gary Kim <gary@garykim.dev> * Return error in sha1CurrentPatternProcessor Co-Authored-By: mrsdizzie <info@mrsdizzie.com> * Import Gitea log module Signed-off-by: Gary Kim <gary@garykim.dev> * Revert "Return error in sha1CurrentPatternProcessor" This reverts commit 28f561cac46ef7e51aa26aefcbe9aca4671366a6. Signed-off-by: Gary Kim <gary@garykim.dev> * Add debug logging to sha1CurrentPatternProcessor This will log errors by the git command run in sha1CurrentPatternProcessor if the error is one that was unexpected. Signed-off-by: Gary Kim <gary@garykim.dev>
2019-08-14 04:04:55 -04:00
}
link := util.URLJoin(setting.AppURL, ctx.Metas["user"], ctx.Metas["repo"], "commit", hash)
replaceContent(node, m[2], m[3], createCodeLink(link, base.ShortSha(hash), "commit"))
start = 0
node = node.NextSibling.NextSibling
}
}
// emailAddressProcessor replaces raw email addresses with a mailto: link.
func emailAddressProcessor(ctx *RenderContext, node *html.Node) {
next := node.NextSibling
for node != nil && node != next {
m := emailRegex.FindStringSubmatchIndex(node.Data)
if m == nil {
return
}
mail := node.Data[m[2]:m[3]]
replaceContent(node, m[2], m[3], createLink("mailto:"+mail, mail, "mailto"))
node = node.NextSibling.NextSibling
}
}
// linkProcessor creates links for any HTTP or HTTPS URL not captured by
// markdown.
func linkProcessor(ctx *RenderContext, node *html.Node) {
next := node.NextSibling
for node != nil && node != next {
m := common.LinkRegex.FindStringIndex(node.Data)
if m == nil {
return
}
uri := node.Data[m[0]:m[1]]
replaceContent(node, m[0], m[1], createLink(uri, uri, "link"))
node = node.NextSibling.NextSibling
}
}
func genDefaultLinkProcessor(defaultLink string) processor {
return func(ctx *RenderContext, node *html.Node) {
ch := &html.Node{
Parent: node,
Type: html.TextNode,
Data: node.Data,
}
node.Type = html.ElementNode
node.Data = "a"
node.DataAtom = atom.A
node.Attr = []html.Attribute{
{Key: "href", Val: defaultLink},
{Key: "class", Val: "default-link muted"},
}
node.FirstChild, node.LastChild = ch, ch
}
}
// descriptionLinkProcessor creates links for DescriptionHTML
func descriptionLinkProcessor(ctx *RenderContext, node *html.Node) {
next := node.NextSibling
for node != nil && node != next {
m := common.LinkRegex.FindStringIndex(node.Data)
if m == nil {
return
}
uri := node.Data[m[0]:m[1]]
replaceContent(node, m[0], m[1], createDescriptionLink(uri, uri))
node = node.NextSibling.NextSibling
}
}
func createDescriptionLink(href, content string) *html.Node {
textNode := &html.Node{
Type: html.TextNode,
Data: content,
}
linkNode := &html.Node{
FirstChild: textNode,
LastChild: textNode,
Type: html.ElementNode,
Data: "a",
DataAtom: atom.A,
Attr: []html.Attribute{
{Key: "href", Val: href},
{Key: "target", Val: "_blank"},
{Key: "rel", Val: "noopener noreferrer"},
},
}
textNode.Parent = linkNode
return linkNode
}