#2185 fall back to use custom chardet lib

.gopmfile

@@ -16,6 +16,7 @@ github.com/go-macaron/toolbox = commit:ab30a81
 github.com/go-sql-driver/mysql = commit:d512f20
 github.com/go-xorm/core = commit:acb6f00
 github.com/go-xorm/xorm = commit:a8fba4d
+github.com/gogits/chardet = commit:2404f77725
 github.com/gogits/git-module = commit:5cd57b9
 github.com/gogits/go-gogs-client = commit:78460e9
 github.com/issue9/identicon = commit:f8c0d2c

README.md

@@ -3,7 +3,7 @@ Gogs - Go Git Service [![Build Status](https://travis-ci.org/gogits/gogs.svg?bra
 
 ![](https://github.com/gogits/gogs/blob/master/public/img/gogs-large-resize.png?raw=true)
 
-##### Current version: 0.8.14
+##### Current version: 0.8.15
 
 | Web | UI  | Preview  |
 |:-------------:|:-------:|:-------:|

gogs.go

@@ -17,7 +17,7 @@ import (
 	"github.com/gogits/gogs/modules/setting"
 )
 
-const APP_VER = "0.8.14.1230"
+const APP_VER = "0.8.15.1231"
 
 func init() {
 	runtime.GOMAXPROCS(runtime.NumCPU())

models/git_diff.go

@@ -246,8 +246,8 @@ func ParsePatch(maxlines int, reader io.Reader) (*Diff, error) {
 				buf.WriteString("\n")
 			}
 		}
-		charsetLabel := base.DetectEncoding(buf.Bytes())
+		charsetLabel, err := base.DetectEncoding(buf.Bytes())
-		if charsetLabel != "UTF-8" {
+		if charsetLabel != "UTF-8" && err == nil {
 			encoding, _ := charset.Lookup(charsetLabel)
 			if encoding != nil {
 				d := encoding.NewDecoder()

modules/base/tool.go

@@ -23,7 +23,8 @@ import (
 	"github.com/Unknwon/com"
 	"github.com/Unknwon/i18n"
 	"github.com/microcosm-cc/bluemonday"
-	"golang.org/x/net/html/charset"
+
+	"github.com/gogits/chardet"
 
 	"github.com/gogits/gogs/modules/avatar"
 	"github.com/gogits/gogs/modules/log"
@@ -53,19 +54,20 @@ func ShortSha(sha1 string) string {
 	return sha1
 }
 
-func DetectEncoding(content []byte) string {
+func DetectEncoding(content []byte) (string, error) {
-	if utf8.Valid(content[:1024]) {
+	if utf8.Valid(content) {
 		log.Debug("Detected encoding: utf-8 (fast)")
-		return "utf-8"
+		return "UTF-8", nil
 	}
 
-	_, name, certain := charset.DetermineEncoding(content, "")
+	result, err := chardet.NewTextDetector().DetectBest(content)
-	if name != "utf-8" && len(setting.Repository.AnsiCharset) > 0 {
+	if result.Charset != "UTF-8" && len(setting.Repository.AnsiCharset) > 0 {
 		log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)
-		return setting.Repository.AnsiCharset
+		return setting.Repository.AnsiCharset, err
 	}
-	log.Debug("Detected encoding: %s (%v)", name, certain)
+
-	return name
+	log.Debug("Detected encoding: %s", result.Charset)
+	return result.Charset, err
 }
 
 func BasicAuthDecode(encoded string) (string, string, error) {

modules/template/template.go

@@ -130,8 +130,10 @@ func Sha1(str string) string {
 }
 
 func ToUtf8WithErr(content []byte) (error, string) {
-	charsetLabel := base.DetectEncoding(content)
+	charsetLabel, err := base.DetectEncoding(content)
-	if charsetLabel == "utf-8" {
+	if err != nil {
+		return err, ""
+	} else if charsetLabel == "UTF-8" {
 		return nil, string(content)
 	}
 

templates/.VERSION

@@ -1 +1 @@
-0.8.14.1230
+0.8.15.1231

templates/base/footer.tmpl

@@ -7,7 +7,7 @@
 	<footer>
 		<div class="ui container">
 			<div class="ui left">
-				© 2015 Gogs {{if (or .ShowFooterVersion .PageIsAdmin)}}{{.i18n.Tr "version"}}: {{AppVer}}{{end}} {{.i18n.Tr "page"}}: <strong>{{LoadTimes .PageStartTime}}</strong> {{.i18n.Tr "template"}}: <strong>{{call .TmplLoadTimes}}</strong>
+				© 2016 Gogs {{if (or .ShowFooterVersion .PageIsAdmin)}}{{.i18n.Tr "version"}}: {{AppVer}}{{end}} {{.i18n.Tr "page"}}: <strong>{{LoadTimes .PageStartTime}}</strong> {{.i18n.Tr "template"}}: <strong>{{call .TmplLoadTimes}}</strong>
 			</div>
 			<div class="ui right links">
 				{{if .ShowFooterBranding}}