2015-02-11 21:58:37 -05:00
// Copyright 2015 The Gogs Authors. All rights reserved.
2019-02-18 11:00:27 -05:00
// Copyright 2017 The Gitea Authors. All rights reserved.
2015-02-11 21:58:37 -05:00
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
2015-01-22 07:49:52 -05:00
package migrations
import (
2015-08-11 11:24:40 -04:00
"bytes"
2015-07-26 10:06:28 -04:00
"encoding/json"
2015-02-11 21:58:37 -05:00
"fmt"
2015-08-11 11:24:40 -04:00
"io/ioutil"
"os"
"path"
2015-10-31 23:18:58 -04:00
"path/filepath"
2015-01-23 02:54:16 -05:00
"strings"
2016-03-09 19:53:30 -05:00
"time"
2015-01-22 07:56:50 -05:00
2015-02-11 21:58:37 -05:00
"github.com/Unknwon/com"
2015-01-22 07:49:52 -05:00
"github.com/go-xorm/xorm"
2016-02-20 18:13:12 -05:00
gouuid "github.com/satori/go.uuid"
2019-01-09 12:22:57 -05:00
ini "gopkg.in/ini.v1"
2015-02-11 21:58:37 -05:00
2018-02-18 13:14:37 -05:00
"code.gitea.io/gitea/modules/generate"
2016-11-10 11:24:48 -05:00
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
2015-01-22 07:49:52 -05:00
)
2016-11-28 10:44:17 -05:00
const minDBVersion = 4
2015-02-11 21:58:37 -05:00
2016-11-28 10:44:17 -05:00
// Migration describes on migration from lower version to high version
2015-02-11 23:10:30 -05:00
type Migration interface {
Description ( ) string
Migrate ( * xorm . Engine ) error
}
type migration struct {
description string
migrate func ( * xorm . Engine ) error
}
2016-11-28 10:44:17 -05:00
// NewMigration creates a new migration
2015-02-11 23:10:30 -05:00
func NewMigration ( desc string , fn func ( * xorm . Engine ) error ) Migration {
return & migration { desc , fn }
}
2016-11-28 10:44:17 -05:00
// Description returns the migration's description
2015-02-11 23:10:30 -05:00
func ( m * migration ) Description ( ) string {
return m . description
}
2016-11-28 10:44:17 -05:00
// Migrate executes the migration
2015-02-11 23:10:30 -05:00
func ( m * migration ) Migrate ( x * xorm . Engine ) error {
return m . migrate ( x )
}
2015-01-22 07:49:52 -05:00
2016-11-28 10:44:17 -05:00
// Version describes the version table. Should have only one row with id==1
2015-01-22 07:49:52 -05:00
type Version struct {
2016-07-15 22:08:04 -04:00
ID int64 ` xorm:"pk autoincr" `
2015-01-22 07:49:52 -05:00
Version int64
}
2017-12-13 09:52:18 -05:00
func emptyMigration ( x * xorm . Engine ) error {
return nil
}
2015-01-22 07:49:52 -05:00
// This is a sequence of migrations. Add new migrations to the bottom of the list.
2015-02-12 12:46:21 -05:00
// If you want to "retire" a migration, remove it from the top of the list and
2016-11-28 10:44:17 -05:00
// update minDBVersion accordingly
2015-02-11 23:10:30 -05:00
var migrations = [ ] Migration {
2016-08-31 07:31:53 -04:00
// v0 -> v4: before 0.6.0 -> 0.7.33
2016-07-07 19:25:09 -04:00
NewMigration ( "fix locale file load panic" , fixLocaleFileLoadPanic ) , // V4 -> V5:v0.6.0
2016-11-28 10:44:17 -05:00
NewMigration ( "trim action compare URL prefix" , trimCommitActionAppURLPrefix ) , // V5 -> V6:v0.6.3
2016-07-07 19:25:09 -04:00
NewMigration ( "generate issue-label from issue" , issueToIssueLabel ) , // V6 -> V7:v0.6.4
NewMigration ( "refactor attachment table" , attachmentRefactor ) , // V7 -> V8:v0.6.4
NewMigration ( "rename pull request fields" , renamePullRequestFields ) , // V8 -> V9:v0.6.16
NewMigration ( "clean up migrate repo info" , cleanUpMigrateRepoInfo ) , // V9 -> V10:v0.6.20
NewMigration ( "generate rands and salt for organizations" , generateOrgRandsAndSalt ) , // V10 -> V11:v0.8.5
NewMigration ( "convert date to unix timestamp" , convertDateToUnix ) , // V11 -> V12:v0.9.2
NewMigration ( "convert LDAP UseSSL option to SecurityProtocol" , ldapUseSSLToSecurityProtocol ) , // V12 -> V13:v0.9.37
2016-08-26 17:07:21 -04:00
// v13 -> v14:v0.9.87
NewMigration ( "set comment updated with created" , setCommentUpdatedWithCreated ) ,
2017-02-04 10:53:46 -05:00
// v14 -> v15
2016-11-12 21:54:04 -05:00
NewMigration ( "create user column diff view style" , createUserColumnDiffViewStyle ) ,
2017-02-04 10:53:46 -05:00
// v15 -> v16
2016-12-30 21:33:30 -05:00
NewMigration ( "create user column allow create organization" , createAllowCreateOrganizationColumn ) ,
2017-02-04 10:53:46 -05:00
// V16 -> v17
NewMigration ( "create repo unit table and add units for all repos" , addUnitsToTables ) ,
2017-02-21 10:02:10 -05:00
// v17 -> v18
NewMigration ( "set protect branches updated with created" , setProtectedBranchUpdatedWithCreated ) ,
2017-02-22 02:14:37 -05:00
// v18 -> v19
NewMigration ( "add external login user" , addExternalLoginUser ) ,
2017-02-22 22:40:44 -05:00
// v19 -> v20
NewMigration ( "generate and migrate Git hooks" , generateAndMigrateGitHooks ) ,
2017-02-25 09:58:57 -05:00
// v20 -> v21
2017-02-25 22:58:02 -05:00
NewMigration ( "use new avatar path name for security reason" , useNewNameAvatars ) ,
2017-03-02 11:36:47 -05:00
// v21 -> v22
NewMigration ( "rewrite authorized_keys file via new format" , useNewPublickeyFormat ) ,
2017-03-14 02:39:02 -04:00
// v22 -> v23
2017-03-13 02:27:29 -04:00
NewMigration ( "generate and migrate wiki Git hooks" , generateAndMigrateWikiGitHooks ) ,
2017-03-17 10:16:08 -04:00
// v23 -> v24
NewMigration ( "add user openid table" , addUserOpenID ) ,
2017-03-17 10:24:51 -04:00
// v24 -> v25
NewMigration ( "change the key_id and primary_key_id type" , changeGPGKeysColumns ) ,
2017-03-20 04:31:08 -04:00
// v25 -> v26
NewMigration ( "add show field in user openid table" , addUserOpenIDShow ) ,
2017-03-22 21:12:51 -04:00
// v26 -> v27
NewMigration ( "generate and migrate repo and wiki Git hooks" , generateAndMigrateGitHookChains ) ,
2017-04-08 11:27:26 -04:00
// v27 -> v28
NewMigration ( "change mirror interval from hours to time.Duration" , convertIntervalToDuration ) ,
2017-04-11 09:30:15 -04:00
// v28 -> v29
NewMigration ( "add field for repo size" , addRepoSize ) ,
2017-04-21 07:32:31 -04:00
// v29 -> v30
NewMigration ( "add commit status table" , addCommitStatus ) ,
2017-05-02 04:41:44 -04:00
// v30 -> 31
NewMigration ( "add primary key to external login user" , addExternalLoginUserPK ) ,
2017-11-28 18:35:23 -05:00
// v31 -> 32
2017-05-10 09:10:18 -04:00
NewMigration ( "add field for login source synchronization" , addLoginSourceSyncEnabledColumn ) ,
2017-05-18 10:54:24 -04:00
// v32 -> v33
NewMigration ( "add units for team" , addUnitsToRepoTeam ) ,
2017-05-25 21:38:18 -04:00
// v33 -> v34
NewMigration ( "remove columns from action" , removeActionColumns ) ,
2017-06-05 06:40:25 -04:00
// v34 -> v35
NewMigration ( "give all units to owner teams" , giveAllUnitsToOwnerTeams ) ,
2017-06-25 14:20:29 -04:00
// v35 -> v36
NewMigration ( "adds comment to an action" , addCommentIDToAction ) ,
2017-07-02 09:50:57 -04:00
// v36 -> v37
NewMigration ( "regenerate git hooks" , regenerateGitHooks36 ) ,
2017-07-12 10:58:52 -04:00
// v37 -> v38
NewMigration ( "unescape user full names" , unescapeUserFullNames ) ,
2017-07-16 22:04:43 -04:00
// v38 -> v39
NewMigration ( "remove commits and settings unit types" , removeCommitsUnitType ) ,
2017-09-12 02:48:13 -04:00
// v39 -> v40
2017-12-13 09:52:18 -05:00
NewMigration ( "add tags to releases and sync existing repositories" , releaseAddColumnIsTagAndSyncTags ) ,
2017-09-14 04:16:22 -04:00
// v40 -> v41
2017-12-13 09:52:18 -05:00
NewMigration ( "fix protected branch can push value to false" , fixProtectedBranchCanPushValue ) ,
2017-09-15 20:18:25 -04:00
// v41 -> v42
2017-12-13 09:52:18 -05:00
NewMigration ( "remove duplicate unit types" , removeDuplicateUnitTypes ) ,
2017-09-20 01:26:49 -04:00
// v42 -> v43
2017-12-13 09:52:18 -05:00
NewMigration ( "empty step" , emptyMigration ) ,
2017-09-20 10:52:23 -04:00
// v43 -> v44
2017-12-13 09:52:18 -05:00
NewMigration ( "empty step" , emptyMigration ) ,
2017-09-28 09:14:51 -04:00
// v44 -> v45
2017-12-13 09:52:18 -05:00
NewMigration ( "empty step" , emptyMigration ) ,
2017-10-02 16:22:25 -04:00
// v45 -> v46
NewMigration ( "remove index column from repo_unit table" , removeIndexColumnFromRepoUnitTable ) ,
2017-10-14 10:37:43 -04:00
// v46 -> v47
NewMigration ( "remove organization watch repositories" , removeOrganizationWatchRepo ) ,
2017-10-25 20:49:16 -04:00
// v47 -> v48
NewMigration ( "add deleted branches" , addDeletedBranch ) ,
2017-10-27 02:10:54 -04:00
// v48 -> v49
NewMigration ( "add repo indexer status" , addRepoIndexerStatus ) ,
2017-11-28 18:35:23 -05:00
// v49 -> v50
2017-12-13 09:52:18 -05:00
NewMigration ( "adds time tracking and stopwatches" , addTimetracking ) ,
2017-12-03 18:14:26 -05:00
// v50 -> v51
2017-12-13 09:52:18 -05:00
NewMigration ( "migrate protected branch struct" , migrateProtectedBranchStruct ) ,
// v51 -> v52
NewMigration ( "add default value to user prohibit_login" , addDefaultValueToUserProhibitLogin ) ,
// v52 -> v53
NewMigration ( "add lfs lock table" , addLFSLock ) ,
// v53 -> v54
2017-12-03 18:14:26 -05:00
NewMigration ( "add reactions" , addReactions ) ,
2018-01-05 13:56:50 -05:00
// v54 -> v55
NewMigration ( "add pull request options" , addPullRequestOptions ) ,
2018-01-06 17:55:53 -05:00
// v55 -> v56
NewMigration ( "add writable deploy keys" , addModeToDeploKeys ) ,
2018-01-08 02:48:37 -05:00
// v56 -> v57
NewMigration ( "remove is_owner, num_teams columns from org_user" , removeIsOwnerColumnFromOrgUser ) ,
2018-02-18 21:39:26 -05:00
// v57 -> v58
NewMigration ( "add closed_unix column for issues" , addIssueClosedTime ) ,
2018-03-12 22:03:55 -04:00
// v58 -> v59
NewMigration ( "add label descriptions" , addLabelsDescriptions ) ,
2018-03-25 06:01:32 -04:00
// v59 -> v60
NewMigration ( "add merge whitelist for protected branches" , addProtectedBranchMergeWhitelist ) ,
2018-03-27 10:13:20 -04:00
// v60 -> v61
NewMigration ( "add is_fsck_enabled column for repos" , addFsckEnabledToRepo ) ,
2018-03-30 21:10:44 -04:00
// v61 -> v62
NewMigration ( "add size column for attachments" , addSizeToAttachment ) ,
2018-05-02 11:02:02 -04:00
// v62 -> v63
NewMigration ( "add last used passcode column for TOTP" , addLastUsedPasscodeTOTP ) ,
2018-05-04 20:28:30 -04:00
// v63 -> v64
NewMigration ( "add language column for user setting" , addLanguageSetting ) ,
2018-05-09 12:29:04 -04:00
// v64 -> v65
NewMigration ( "add multiple assignees" , addMultipleAssignees ) ,
2018-05-19 10:12:37 -04:00
// v65 -> v66
NewMigration ( "add u2f" , addU2FReg ) ,
2018-05-24 00:59:02 -04:00
// v66 -> v67
NewMigration ( "add login source id column for public_key table" , addLoginSourceIDToPublicKeyTable ) ,
2018-06-19 15:44:33 -04:00
// v67 -> v68
NewMigration ( "remove stale watches" , removeStaleWatches ) ,
2018-06-21 05:09:46 -04:00
// v68 -> V69
NewMigration ( "Reformat and remove incorrect topics" , reformatAndRemoveIncorrectTopics ) ,
2018-06-21 12:00:13 -04:00
// v69 -> v70
NewMigration ( "move team units to team_unit table" , moveTeamUnitsToTeamUnitTable ) ,
2018-07-17 17:23:58 -04:00
// v70 -> v71
NewMigration ( "add issue_dependencies" , addIssueDependencies ) ,
2018-08-06 00:43:22 -04:00
// v71 -> v72
2018-07-27 08:54:50 -04:00
NewMigration ( "protect each scratch token" , addScratchHash ) ,
2018-08-06 00:43:22 -04:00
// v72 -> v73
NewMigration ( "add review" , addReview ) ,
2018-09-13 08:04:25 -04:00
// v73 -> v74
NewMigration ( "add must_change_password column for users table" , addMustChangePassword ) ,
2018-12-11 06:28:37 -05:00
// v74 -> v75
NewMigration ( "add approval whitelists to protected branches" , addApprovalWhitelistsToProtectedBranches ) ,
2018-12-18 11:26:26 -05:00
// v75 -> v76
NewMigration ( "clear nonused data which not deleted when user was deleted" , clearNonusedData ) ,
2018-12-27 05:27:08 -05:00
// v76 -> v77
NewMigration ( "add pull request rebase with merge commit" , addPullRequestRebaseWithMerge ) ,
2019-01-09 12:22:57 -05:00
// v77 -> v78
NewMigration ( "add theme to users" , addUserDefaultTheme ) ,
2019-01-17 19:01:04 -05:00
// v78 -> v79
NewMigration ( "rename repo is_bare to repo is_empty" , renameRepoIsBareToIsEmpty ) ,
2019-02-10 14:27:19 -05:00
// v79 -> v80
NewMigration ( "add can close issues via commit in any branch" , addCanCloseIssuesViaCommitInAnyBranch ) ,
2019-02-18 15:55:04 -05:00
// v80 -> v81
NewMigration ( "add is locked to issues" , addIsLockedToIssues ) ,
2019-03-04 21:34:52 -05:00
// v81 -> v82
NewMigration ( "update U2F counter type" , changeU2FCounterType ) ,
2019-03-10 23:44:58 -04:00
// v82 -> v83
NewMigration ( "hot fix for wrong release sha1 on release table" , fixReleaseSha1OnReleaseTable ) ,
2019-04-02 15:25:05 -04:00
// v83 -> v84
NewMigration ( "add uploader id for table attachment" , addUploaderIDForAttachment ) ,
2019-04-14 12:43:56 -04:00
// v84 -> v85
NewMigration ( "add table to store original imported gpg keys" , addGPGKeyImport ) ,
2019-05-04 11:45:34 -04:00
// v85 -> v86
NewMigration ( "hash application token" , hashAppToken ) ,
2015-01-23 02:54:16 -05:00
}
2015-01-22 07:49:52 -05:00
// Migrate database to current version
func Migrate ( x * xorm . Engine ) error {
2015-01-22 08:01:45 -05:00
if err := x . Sync ( new ( Version ) ) ; err != nil {
2015-02-11 21:58:37 -05:00
return fmt . Errorf ( "sync: %v" , err )
2015-01-22 08:01:45 -05:00
}
2015-01-22 07:49:52 -05:00
2016-07-15 22:08:04 -04:00
currentVersion := & Version { ID : 1 }
2015-01-22 07:49:52 -05:00
has , err := x . Get ( currentVersion )
if err != nil {
2015-02-11 21:58:37 -05:00
return fmt . Errorf ( "get: %v" , err )
2015-01-22 07:56:50 -05:00
} else if ! has {
2015-12-10 19:52:06 -05:00
// If the version record does not exist we think
// it is a fresh installation and we can skip all migrations.
2016-12-23 20:37:35 -05:00
currentVersion . ID = 0
2016-11-28 10:44:17 -05:00
currentVersion . Version = int64 ( minDBVersion + len ( migrations ) )
2015-01-23 02:54:16 -05:00
2015-01-22 07:56:50 -05:00
if _ , err = x . InsertOne ( currentVersion ) ; err != nil {
2015-02-11 21:58:37 -05:00
return fmt . Errorf ( "insert: %v" , err )
2015-01-22 07:56:50 -05:00
}
2015-01-22 07:49:52 -05:00
}
v := currentVersion . Version
2016-11-28 10:44:17 -05:00
if minDBVersion > v {
2019-04-02 03:48:31 -04:00
log . Fatal ( ` Gitea no longer supports auto - migration from your previously installed version .
2015-12-10 19:52:06 -05:00
Please try to upgrade to a lower version ( >= v0 .6 .0 ) first , then upgrade to current version . ` )
2015-11-25 09:27:27 -05:00
return nil
}
2016-11-28 10:44:17 -05:00
if int ( v - minDBVersion ) > len ( migrations ) {
2016-12-28 03:33:21 -05:00
// User downgraded Gitea.
2016-11-28 10:44:17 -05:00
currentVersion . Version = int64 ( len ( migrations ) + minDBVersion )
2017-10-05 00:43:04 -04:00
_ , err = x . ID ( 1 ) . Update ( currentVersion )
2015-08-11 11:24:40 -04:00
return err
2015-08-10 10:59:12 -04:00
}
2016-11-28 10:44:17 -05:00
for i , m := range migrations [ v - minDBVersion : ] {
2015-02-11 23:10:30 -05:00
log . Info ( "Migration: %s" , m . Description ( ) )
if err = m . Migrate ( x ) ; err != nil {
return fmt . Errorf ( "do migrate: %v" , err )
2015-01-22 07:49:52 -05:00
}
currentVersion . Version = v + int64 ( i ) + 1
2017-10-05 00:43:04 -04:00
if _ , err = x . ID ( 1 ) . Update ( currentVersion ) ; err != nil {
2015-01-22 08:01:45 -05:00
return err
}
2015-01-22 07:49:52 -05:00
}
return nil
}
2018-05-09 12:29:04 -04:00
func dropTableColumns ( sess * xorm . Session , tableName string , columnNames ... string ) ( err error ) {
2018-03-07 01:44:12 -05:00
if tableName == "" || len ( columnNames ) == 0 {
return nil
}
switch {
case setting . UseSQLite3 :
log . Warn ( "Unable to drop columns in SQLite" )
case setting . UseMySQL , setting . UseTiDB , setting . UsePostgreSQL :
cols := ""
for _ , col := range columnNames {
if cols != "" {
cols += ", "
}
cols += "DROP COLUMN `" + col + "`"
}
2018-05-09 12:29:04 -04:00
if _ , err := sess . Exec ( fmt . Sprintf ( "ALTER TABLE `%s` %s" , tableName , cols ) ) ; err != nil {
2018-03-07 01:44:12 -05:00
return fmt . Errorf ( "Drop table `%s` columns %v: %v" , tableName , columnNames , err )
}
case setting . UseMSSQL :
cols := ""
for _ , col := range columnNames {
if cols != "" {
cols += ", "
}
cols += "`" + strings . ToLower ( col ) + "`"
}
sql := fmt . Sprintf ( "SELECT Name FROM SYS.DEFAULT_CONSTRAINTS WHERE PARENT_OBJECT_ID = OBJECT_ID('%[1]s') AND PARENT_COLUMN_ID IN (SELECT column_id FROM sys.columns WHERE lower(NAME) IN (%[2]s) AND object_id = OBJECT_ID('%[1]s'))" ,
tableName , strings . Replace ( cols , "`" , "'" , - 1 ) )
constraints := make ( [ ] string , 0 )
if err := sess . SQL ( sql ) . Find ( & constraints ) ; err != nil {
sess . Rollback ( )
return fmt . Errorf ( "Find constraints: %v" , err )
}
for _ , constraint := range constraints {
if _ , err := sess . Exec ( fmt . Sprintf ( "ALTER TABLE `%s` DROP CONSTRAINT `%s`" , tableName , constraint ) ) ; err != nil {
sess . Rollback ( )
return fmt . Errorf ( "Drop table `%s` constraint `%s`: %v" , tableName , constraint , err )
}
}
if _ , err := sess . Exec ( fmt . Sprintf ( "ALTER TABLE `%s` DROP COLUMN %s" , tableName , cols ) ) ; err != nil {
sess . Rollback ( )
return fmt . Errorf ( "Drop table `%s` columns %v: %v" , tableName , columnNames , err )
}
return sess . Commit ( )
default :
2019-04-02 03:48:31 -04:00
log . Fatal ( "Unrecognized DB" )
2018-03-07 01:44:12 -05:00
}
return nil
}
2015-03-25 19:51:22 -04:00
func fixLocaleFileLoadPanic ( _ * xorm . Engine ) error {
cfg , err := ini . Load ( setting . CustomConf )
if err != nil {
return fmt . Errorf ( "load custom config: %v" , err )
}
cfg . DeleteSection ( "i18n" )
if err = cfg . SaveTo ( setting . CustomConf ) ; err != nil {
return fmt . Errorf ( "save custom config: %v" , err )
}
setting . Langs = strings . Split ( strings . Replace ( strings . Join ( setting . Langs , "," ) , "fr-CA" , "fr-FR" , 1 ) , "," )
return nil
}
2015-07-26 10:06:28 -04:00
2016-11-28 10:44:17 -05:00
func trimCommitActionAppURLPrefix ( x * xorm . Engine ) error {
2015-07-26 10:06:28 -04:00
type PushCommit struct {
Sha1 string
Message string
AuthorEmail string
AuthorName string
}
type PushCommits struct {
Len int
Commits [ ] * PushCommit
2016-11-28 10:44:17 -05:00
CompareURL string ` json:"CompareUrl" `
2015-07-26 10:06:28 -04:00
}
type Action struct {
ID int64 ` xorm:"pk autoincr" `
Content string ` xorm:"TEXT" `
}
results , err := x . Query ( "SELECT `id`,`content` FROM `action` WHERE `op_type`=?" , 5 )
if err != nil {
return fmt . Errorf ( "select commit actions: %v" , err )
}
sess := x . NewSession ( )
2017-06-20 20:57:05 -04:00
defer sess . Close ( )
2015-07-26 10:06:28 -04:00
if err = sess . Begin ( ) ; err != nil {
return err
}
var pushCommits * PushCommits
for _ , action := range results {
actID := com . StrTo ( string ( action [ "id" ] ) ) . MustInt64 ( )
if actID == 0 {
continue
}
pushCommits = new ( PushCommits )
if err = json . Unmarshal ( action [ "content" ] , pushCommits ) ; err != nil {
2015-11-08 14:31:49 -05:00
return fmt . Errorf ( "unmarshal action content[%d]: %v" , actID , err )
2015-07-26 10:06:28 -04:00
}
2016-11-28 10:44:17 -05:00
infos := strings . Split ( pushCommits . CompareURL , "/" )
2015-07-26 10:06:28 -04:00
if len ( infos ) <= 4 {
continue
}
2016-11-28 10:44:17 -05:00
pushCommits . CompareURL = strings . Join ( infos [ len ( infos ) - 4 : ] , "/" )
2015-07-26 10:06:28 -04:00
p , err := json . Marshal ( pushCommits )
if err != nil {
2015-11-08 14:31:49 -05:00
return fmt . Errorf ( "marshal action content[%d]: %v" , actID , err )
2015-07-26 10:06:28 -04:00
}
if _ , err = sess . Id ( actID ) . Update ( & Action {
Content : string ( p ) ,
} ) ; err != nil {
2015-08-19 08:08:57 -04:00
return fmt . Errorf ( "update action[%d]: %v" , actID , err )
2015-07-26 10:06:28 -04:00
}
}
return sess . Commit ( )
}
2015-08-10 02:42:50 -04:00
func issueToIssueLabel ( x * xorm . Engine ) error {
type IssueLabel struct {
ID int64 ` xorm:"pk autoincr" `
IssueID int64 ` xorm:"UNIQUE(s)" `
LabelID int64 ` xorm:"UNIQUE(s)" `
}
issueLabels := make ( [ ] * IssueLabel , 0 , 50 )
results , err := x . Query ( "SELECT `id`,`label_ids` FROM `issue`" )
if err != nil {
2015-12-10 19:52:06 -05:00
if strings . Contains ( err . Error ( ) , "no such column" ) ||
strings . Contains ( err . Error ( ) , "Unknown column" ) {
2015-08-10 02:42:50 -04:00
return nil
}
return fmt . Errorf ( "select issues: %v" , err )
}
for _ , issue := range results {
issueID := com . StrTo ( issue [ "id" ] ) . MustInt64 ( )
// Just in case legacy code can have duplicated IDs for same label.
mark := make ( map [ int64 ] bool )
for _ , idStr := range strings . Split ( string ( issue [ "label_ids" ] ) , "|" ) {
labelID := com . StrTo ( strings . TrimPrefix ( idStr , "$" ) ) . MustInt64 ( )
if labelID == 0 || mark [ labelID ] {
continue
}
mark [ labelID ] = true
issueLabels = append ( issueLabels , & IssueLabel {
IssueID : issueID ,
LabelID : labelID ,
} )
}
}
sess := x . NewSession ( )
2017-06-20 20:57:05 -04:00
defer sess . Close ( )
2015-08-10 02:42:50 -04:00
if err = sess . Begin ( ) ; err != nil {
return err
}
if err = sess . Sync2 ( new ( IssueLabel ) ) ; err != nil {
2016-08-26 20:32:41 -04:00
return fmt . Errorf ( "Sync2: %v" , err )
2015-08-10 02:42:50 -04:00
} else if _ , err = sess . Insert ( issueLabels ) ; err != nil {
return fmt . Errorf ( "insert issue-labels: %v" , err )
}
return sess . Commit ( )
}
2015-08-11 11:24:40 -04:00
func attachmentRefactor ( x * xorm . Engine ) error {
type Attachment struct {
ID int64 ` xorm:"pk autoincr" `
UUID string ` xorm:"uuid INDEX" `
// For rename purpose.
Path string ` xorm:"-" `
NewPath string ` xorm:"-" `
}
results , err := x . Query ( "SELECT * FROM `attachment`" )
if err != nil {
return fmt . Errorf ( "select attachments: %v" , err )
}
attachments := make ( [ ] * Attachment , 0 , len ( results ) )
for _ , attach := range results {
if ! com . IsExist ( string ( attach [ "path" ] ) ) {
// If the attachment is already missing, there is no point to update it.
continue
}
attachments = append ( attachments , & Attachment {
ID : com . StrTo ( attach [ "id" ] ) . MustInt64 ( ) ,
UUID : gouuid . NewV4 ( ) . String ( ) ,
Path : string ( attach [ "path" ] ) ,
} )
}
sess := x . NewSession ( )
2017-06-20 20:57:05 -04:00
defer sess . Close ( )
2015-08-11 11:24:40 -04:00
if err = sess . Begin ( ) ; err != nil {
return err
}
if err = sess . Sync2 ( new ( Attachment ) ) ; err != nil {
return fmt . Errorf ( "Sync2: %v" , err )
}
// Note: Roll back for rename can be a dead loop,
// so produces a backup file.
var buf bytes . Buffer
buf . WriteString ( "# old path -> new path\n" )
// Update database first because this is where error happens the most often.
for _ , attach := range attachments {
if _ , err = sess . Id ( attach . ID ) . Update ( attach ) ; err != nil {
return err
}
attach . NewPath = path . Join ( setting . AttachmentPath , attach . UUID [ 0 : 1 ] , attach . UUID [ 1 : 2 ] , attach . UUID )
buf . WriteString ( attach . Path )
buf . WriteString ( "\t" )
buf . WriteString ( attach . NewPath )
buf . WriteString ( "\n" )
}
// Then rename attachments.
isSucceed := true
defer func ( ) {
if isSucceed {
return
}
dumpPath := path . Join ( setting . LogRootPath , "attachment_path.dump" )
ioutil . WriteFile ( dumpPath , buf . Bytes ( ) , 0666 )
2017-01-29 15:13:57 -05:00
log . Info ( "Failed to rename some attachments, old and new paths are saved into: %s" , dumpPath )
2015-08-11 11:24:40 -04:00
} ( )
for _ , attach := range attachments {
if err = os . MkdirAll ( path . Dir ( attach . NewPath ) , os . ModePerm ) ; err != nil {
isSucceed = false
return err
}
if err = os . Rename ( attach . Path , attach . NewPath ) ; err != nil {
isSucceed = false
return err
}
}
return sess . Commit ( )
}
2015-10-18 19:30:39 -04:00
func renamePullRequestFields ( x * xorm . Engine ) ( err error ) {
type PullRequest struct {
ID int64 ` xorm:"pk autoincr" `
PullID int64 ` xorm:"INDEX" `
PullIndex int64
HeadBarcnh string
IssueID int64 ` xorm:"INDEX" `
Index int64
HeadBranch string
}
if err = x . Sync ( new ( PullRequest ) ) ; err != nil {
return fmt . Errorf ( "sync: %v" , err )
}
results , err := x . Query ( "SELECT `id`,`pull_id`,`pull_index`,`head_barcnh` FROM `pull_request`" )
if err != nil {
if strings . Contains ( err . Error ( ) , "no such column" ) {
return nil
}
return fmt . Errorf ( "select pull requests: %v" , err )
}
sess := x . NewSession ( )
2017-06-20 20:57:05 -04:00
defer sess . Close ( )
2015-10-18 19:30:39 -04:00
if err = sess . Begin ( ) ; err != nil {
return err
}
var pull * PullRequest
for _ , pr := range results {
pull = & PullRequest {
ID : com . StrTo ( pr [ "id" ] ) . MustInt64 ( ) ,
IssueID : com . StrTo ( pr [ "pull_id" ] ) . MustInt64 ( ) ,
Index : com . StrTo ( pr [ "pull_index" ] ) . MustInt64 ( ) ,
HeadBranch : string ( pr [ "head_barcnh" ] ) ,
}
2015-12-10 19:52:06 -05:00
if pull . Index == 0 {
continue
}
2015-10-18 19:30:39 -04:00
if _ , err = sess . Id ( pull . ID ) . Update ( pull ) ; err != nil {
return err
}
}
return sess . Commit ( )
}
2015-10-31 23:18:58 -04:00
func cleanUpMigrateRepoInfo ( x * xorm . Engine ) ( err error ) {
type (
User struct {
ID int64 ` xorm:"pk autoincr" `
LowerName string
}
Repository struct {
ID int64 ` xorm:"pk autoincr" `
OwnerID int64
LowerName string
}
)
repos := make ( [ ] * Repository , 0 , 25 )
if err = x . Where ( "is_mirror=?" , false ) . Find ( & repos ) ; err != nil {
return fmt . Errorf ( "select all non-mirror repositories: %v" , err )
}
var user * User
for _ , repo := range repos {
user = & User { ID : repo . OwnerID }
has , err := x . Get ( user )
if err != nil {
return fmt . Errorf ( "get owner of repository[%d - %d]: %v" , repo . ID , repo . OwnerID , err )
} else if ! has {
continue
}
configPath := filepath . Join ( setting . RepoRootPath , user . LowerName , repo . LowerName + ".git/config" )
2015-10-31 23:25:08 -04:00
// In case repository file is somehow missing.
if ! com . IsFile ( configPath ) {
continue
}
2015-10-31 23:18:58 -04:00
cfg , err := ini . Load ( configPath )
if err != nil {
return fmt . Errorf ( "open config file: %v" , err )
}
cfg . DeleteSection ( "remote \"origin\"" )
if err = cfg . SaveToIndent ( configPath , "\t" ) ; err != nil {
return fmt . Errorf ( "save config file: %v" , err )
}
}
return nil
}
2015-12-14 17:06:54 -05:00
func generateOrgRandsAndSalt ( x * xorm . Engine ) ( err error ) {
type User struct {
ID int64 ` xorm:"pk autoincr" `
Rands string ` xorm:"VARCHAR(10)" `
Salt string ` xorm:"VARCHAR(10)" `
}
orgs := make ( [ ] * User , 0 , 10 )
if err = x . Where ( "type=1" ) . And ( "rands=''" ) . Find ( & orgs ) ; err != nil {
return fmt . Errorf ( "select all organizations: %v" , err )
}
sess := x . NewSession ( )
2017-06-20 20:57:05 -04:00
defer sess . Close ( )
2015-12-14 17:06:54 -05:00
if err = sess . Begin ( ) ; err != nil {
return err
}
for _ , org := range orgs {
2018-02-18 13:14:37 -05:00
if org . Rands , err = generate . GetRandomString ( 10 ) ; err != nil {
2016-12-20 07:32:02 -05:00
return err
}
2018-02-18 13:14:37 -05:00
if org . Salt , err = generate . GetRandomString ( 10 ) ; err != nil {
2016-12-20 07:32:02 -05:00
return err
}
2015-12-14 17:06:54 -05:00
if _ , err = sess . Id ( org . ID ) . Update ( org ) ; err != nil {
return err
}
}
return sess . Commit ( )
}
2016-03-09 19:53:30 -05:00
2016-11-28 10:44:17 -05:00
// TAction defines the struct for migrating table action
2016-03-09 19:53:30 -05:00
type TAction struct {
ID int64 ` xorm:"pk autoincr" `
CreatedUnix int64
}
2016-11-28 10:44:17 -05:00
// TableName will be invoked by XORM to customrize the table name
2016-03-09 19:53:30 -05:00
func ( t * TAction ) TableName ( ) string { return "action" }
2016-11-28 10:44:17 -05:00
// TNotice defines the struct for migrating table notice
2016-03-09 19:53:30 -05:00
type TNotice struct {
ID int64 ` xorm:"pk autoincr" `
CreatedUnix int64
}
2016-11-28 10:44:17 -05:00
// TableName will be invoked by XORM to customrize the table name
2016-03-09 19:53:30 -05:00
func ( t * TNotice ) TableName ( ) string { return "notice" }
2016-11-28 10:44:17 -05:00
// TComment defines the struct for migrating table comment
2016-03-09 19:53:30 -05:00
type TComment struct {
ID int64 ` xorm:"pk autoincr" `
CreatedUnix int64
}
2016-11-28 10:44:17 -05:00
// TableName will be invoked by XORM to customrize the table name
2016-03-09 19:53:30 -05:00
func ( t * TComment ) TableName ( ) string { return "comment" }
2016-11-28 10:44:17 -05:00
// TIssue defines the struct for migrating table issue
2016-03-09 19:53:30 -05:00
type TIssue struct {
ID int64 ` xorm:"pk autoincr" `
DeadlineUnix int64
CreatedUnix int64
UpdatedUnix int64
}
2016-11-28 10:44:17 -05:00
// TableName will be invoked by XORM to customrize the table name
2016-03-09 19:53:30 -05:00
func ( t * TIssue ) TableName ( ) string { return "issue" }
2016-11-28 10:44:17 -05:00
// TMilestone defines the struct for migrating table milestone
2016-03-09 19:53:30 -05:00
type TMilestone struct {
ID int64 ` xorm:"pk autoincr" `
DeadlineUnix int64
ClosedDateUnix int64
}
2016-11-28 10:44:17 -05:00
// TableName will be invoked by XORM to customrize the table name
2016-03-09 19:53:30 -05:00
func ( t * TMilestone ) TableName ( ) string { return "milestone" }
2016-11-28 10:44:17 -05:00
// TAttachment defines the struct for migrating table attachment
2016-03-09 19:53:30 -05:00
type TAttachment struct {
ID int64 ` xorm:"pk autoincr" `
CreatedUnix int64
}
2016-11-28 10:44:17 -05:00
// TableName will be invoked by XORM to customrize the table name
2016-03-09 19:53:30 -05:00
func ( t * TAttachment ) TableName ( ) string { return "attachment" }
2016-11-28 10:44:17 -05:00
// TLoginSource defines the struct for migrating table login_source
2016-03-09 19:53:30 -05:00
type TLoginSource struct {
ID int64 ` xorm:"pk autoincr" `
CreatedUnix int64
UpdatedUnix int64
}
2016-11-28 10:44:17 -05:00
// TableName will be invoked by XORM to customrize the table name
2016-03-09 19:53:30 -05:00
func ( t * TLoginSource ) TableName ( ) string { return "login_source" }
2016-11-28 10:44:17 -05:00
// TPull defines the struct for migrating table pull_request
2016-03-09 19:53:30 -05:00
type TPull struct {
ID int64 ` xorm:"pk autoincr" `
MergedUnix int64
}
2016-11-28 10:44:17 -05:00
// TableName will be invoked by XORM to customrize the table name
2016-03-09 19:53:30 -05:00
func ( t * TPull ) TableName ( ) string { return "pull_request" }
2016-11-28 10:44:17 -05:00
// TRelease defines the struct for migrating table release
2016-03-09 19:53:30 -05:00
type TRelease struct {
ID int64 ` xorm:"pk autoincr" `
CreatedUnix int64
}
2016-11-28 10:44:17 -05:00
// TableName will be invoked by XORM to customrize the table name
2016-03-09 19:53:30 -05:00
func ( t * TRelease ) TableName ( ) string { return "release" }
2016-11-28 10:44:17 -05:00
// TRepo defines the struct for migrating table repository
2016-03-09 19:53:30 -05:00
type TRepo struct {
ID int64 ` xorm:"pk autoincr" `
CreatedUnix int64
UpdatedUnix int64
}
2016-11-28 10:44:17 -05:00
// TableName will be invoked by XORM to customrize the table name
2016-03-09 19:53:30 -05:00
func ( t * TRepo ) TableName ( ) string { return "repository" }
2016-11-28 10:44:17 -05:00
// TMirror defines the struct for migrating table mirror
2016-03-09 19:53:30 -05:00
type TMirror struct {
ID int64 ` xorm:"pk autoincr" `
UpdatedUnix int64
NextUpdateUnix int64
}
2016-11-28 10:44:17 -05:00
// TableName will be invoked by XORM to customrize the table name
2016-03-09 19:53:30 -05:00
func ( t * TMirror ) TableName ( ) string { return "mirror" }
2016-11-28 10:44:17 -05:00
// TPublicKey defines the struct for migrating table public_key
2016-03-09 19:53:30 -05:00
type TPublicKey struct {
ID int64 ` xorm:"pk autoincr" `
CreatedUnix int64
UpdatedUnix int64
}
2016-11-28 10:44:17 -05:00
// TableName will be invoked by XORM to customrize the table name
2016-03-09 19:53:30 -05:00
func ( t * TPublicKey ) TableName ( ) string { return "public_key" }
2016-11-28 10:44:17 -05:00
// TDeployKey defines the struct for migrating table deploy_key
2016-03-09 19:53:30 -05:00
type TDeployKey struct {
ID int64 ` xorm:"pk autoincr" `
CreatedUnix int64
UpdatedUnix int64
}
2016-11-28 10:44:17 -05:00
// TableName will be invoked by XORM to customrize the table name
2016-03-09 19:53:30 -05:00
func ( t * TDeployKey ) TableName ( ) string { return "deploy_key" }
2016-11-28 10:44:17 -05:00
// TAccessToken defines the struct for migrating table access_token
2016-03-09 19:53:30 -05:00
type TAccessToken struct {
ID int64 ` xorm:"pk autoincr" `
CreatedUnix int64
UpdatedUnix int64
}
2016-11-28 10:44:17 -05:00
// TableName will be invoked by XORM to customrize the table name
2016-03-09 19:53:30 -05:00
func ( t * TAccessToken ) TableName ( ) string { return "access_token" }
2016-11-28 10:44:17 -05:00
// TUser defines the struct for migrating table user
2016-03-09 19:53:30 -05:00
type TUser struct {
ID int64 ` xorm:"pk autoincr" `
CreatedUnix int64
UpdatedUnix int64
}
2016-11-28 10:44:17 -05:00
// TableName will be invoked by XORM to customrize the table name
2016-03-09 19:53:30 -05:00
func ( t * TUser ) TableName ( ) string { return "user" }
2016-11-28 10:44:17 -05:00
// TWebhook defines the struct for migrating table webhook
2016-03-09 19:53:30 -05:00
type TWebhook struct {
ID int64 ` xorm:"pk autoincr" `
CreatedUnix int64
UpdatedUnix int64
}
2016-11-28 10:44:17 -05:00
// TableName will be invoked by XORM to customrize the table name
2016-03-09 19:53:30 -05:00
func ( t * TWebhook ) TableName ( ) string { return "webhook" }
func convertDateToUnix ( x * xorm . Engine ) ( err error ) {
2016-07-07 19:25:09 -04:00
log . Info ( "This migration could take up to minutes, please be patient." )
2016-03-09 19:53:30 -05:00
type Bean struct {
ID int64 ` xorm:"pk autoincr" `
Created time . Time
Updated time . Time
Merged time . Time
Deadline time . Time
ClosedDate time . Time
NextUpdate time . Time
}
var tables = [ ] struct {
name string
cols [ ] string
bean interface { }
} {
{ "action" , [ ] string { "created" } , new ( TAction ) } ,
{ "notice" , [ ] string { "created" } , new ( TNotice ) } ,
{ "comment" , [ ] string { "created" } , new ( TComment ) } ,
{ "issue" , [ ] string { "deadline" , "created" , "updated" } , new ( TIssue ) } ,
{ "milestone" , [ ] string { "deadline" , "closed_date" } , new ( TMilestone ) } ,
{ "attachment" , [ ] string { "created" } , new ( TAttachment ) } ,
{ "login_source" , [ ] string { "created" , "updated" } , new ( TLoginSource ) } ,
{ "pull_request" , [ ] string { "merged" } , new ( TPull ) } ,
{ "release" , [ ] string { "created" } , new ( TRelease ) } ,
{ "repository" , [ ] string { "created" , "updated" } , new ( TRepo ) } ,
{ "mirror" , [ ] string { "updated" , "next_update" } , new ( TMirror ) } ,
{ "public_key" , [ ] string { "created" , "updated" } , new ( TPublicKey ) } ,
{ "deploy_key" , [ ] string { "created" , "updated" } , new ( TDeployKey ) } ,
{ "access_token" , [ ] string { "created" , "updated" } , new ( TAccessToken ) } ,
{ "user" , [ ] string { "created" , "updated" } , new ( TUser ) } ,
{ "webhook" , [ ] string { "created" , "updated" } , new ( TWebhook ) } ,
}
for _ , table := range tables {
log . Info ( "Converting table: %s" , table . name )
if err = x . Sync2 ( table . bean ) ; err != nil {
return fmt . Errorf ( "Sync [table: %s]: %v" , table . name , err )
}
offset := 0
for {
beans := make ( [ ] * Bean , 0 , 100 )
2017-08-04 00:42:51 -04:00
if err = x . Table ( table . name ) . Asc ( "id" ) . Limit ( 100 , offset ) . Find ( & beans ) ; err != nil {
2016-03-09 19:53:30 -05:00
return fmt . Errorf ( "select beans [table: %s, offset: %d]: %v" , table . name , offset , err )
}
log . Trace ( "Table [%s]: offset: %d, beans: %d" , table . name , offset , len ( beans ) )
if len ( beans ) == 0 {
break
}
offset += 100
baseSQL := "UPDATE `" + table . name + "` SET "
for _ , bean := range beans {
valSQLs := make ( [ ] string , 0 , len ( table . cols ) )
for _ , col := range table . cols {
fieldSQL := ""
fieldSQL += col + "_unix = "
switch col {
case "deadline" :
if bean . Deadline . IsZero ( ) {
continue
}
2016-07-23 08:24:44 -04:00
fieldSQL += com . ToStr ( bean . Deadline . Unix ( ) )
2016-03-09 19:53:30 -05:00
case "created" :
2016-07-23 08:24:44 -04:00
fieldSQL += com . ToStr ( bean . Created . Unix ( ) )
2016-03-09 19:53:30 -05:00
case "updated" :
2016-07-23 08:24:44 -04:00
fieldSQL += com . ToStr ( bean . Updated . Unix ( ) )
2016-03-09 19:53:30 -05:00
case "closed_date" :
2016-07-23 08:24:44 -04:00
fieldSQL += com . ToStr ( bean . ClosedDate . Unix ( ) )
2016-03-09 19:53:30 -05:00
case "merged" :
2016-07-23 08:24:44 -04:00
fieldSQL += com . ToStr ( bean . Merged . Unix ( ) )
2016-03-09 19:53:30 -05:00
case "next_update" :
2016-07-23 08:24:44 -04:00
fieldSQL += com . ToStr ( bean . NextUpdate . Unix ( ) )
2016-03-09 19:53:30 -05:00
}
valSQLs = append ( valSQLs , fieldSQL )
}
if len ( valSQLs ) == 0 {
continue
}
if _ , err = x . Exec ( baseSQL + strings . Join ( valSQLs , "," ) + " WHERE id = " + com . ToStr ( bean . ID ) ) ; err != nil {
return fmt . Errorf ( "update bean [table: %s, id: %d]: %v" , table . name , bean . ID , err )
}
}
}
}
return nil
}