migrations.go 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390
  1. // Copyright 2015 The Gogs Authors. All rights reserved.
  2. // Use of this source code is governed by a MIT-style
  3. // license that can be found in the LICENSE file.
  4. package migrations
  5. import (
  6. "fmt"
  7. "strings"
  8. "time"
  9. "github.com/unknwon/com"
  10. log "unknwon.dev/clog/v2"
  11. "xorm.io/xorm"
  12. "gogs.io/gogs/internal/tool"
  13. )
  14. const _MIN_DB_VER = 10
  15. type Migration interface {
  16. Description() string
  17. Migrate(*xorm.Engine) error
  18. }
  19. type migration struct {
  20. description string
  21. migrate func(*xorm.Engine) error
  22. }
  23. func NewMigration(desc string, fn func(*xorm.Engine) error) Migration {
  24. return &migration{desc, fn}
  25. }
  26. func (m *migration) Description() string {
  27. return m.description
  28. }
  29. func (m *migration) Migrate(x *xorm.Engine) error {
  30. return m.migrate(x)
  31. }
  32. // The version table. Should have only one row with id==1
  33. type Version struct {
  34. ID int64
  35. Version int64
  36. }
  37. // This is a sequence of migrations. Add new migrations to the bottom of the list.
  38. // If you want to "retire" a migration, remove it from the top of the list and
  39. // update _MIN_VER_DB accordingly
  40. var migrations = []Migration{
  41. // v0 -> v4 : before 0.6.0 -> last support 0.7.33
  42. // v4 -> v10: before 0.7.0 -> last support 0.9.141
  43. NewMigration("generate rands and salt for organizations", generateOrgRandsAndSalt), // V10 -> V11:v0.8.5
  44. NewMigration("convert date to unix timestamp", convertDateToUnix), // V11 -> V12:v0.9.2
  45. NewMigration("convert LDAP UseSSL option to SecurityProtocol", ldapUseSSLToSecurityProtocol), // V12 -> V13:v0.9.37
  46. // v13 -> v14:v0.9.87
  47. NewMigration("set comment updated with created", setCommentUpdatedWithCreated),
  48. // v14 -> v15:v0.9.147
  49. NewMigration("generate and migrate Git hooks", generateAndMigrateGitHooks),
  50. // v15 -> v16:v0.10.16
  51. NewMigration("update repository sizes", updateRepositorySizes),
  52. // v16 -> v17:v0.10.31
  53. NewMigration("remove invalid protect branch whitelist", removeInvalidProtectBranchWhitelist),
  54. // v17 -> v18:v0.11.48
  55. NewMigration("store long text in repository description field", updateRepositoryDescriptionField),
  56. // v18 -> v19:v0.11.55
  57. NewMigration("clean unlinked webhook and hook_tasks", cleanUnlinkedWebhookAndHookTasks),
  58. }
  59. // Migrate database to current version
  60. func Migrate(x *xorm.Engine) error {
  61. if err := x.Sync(new(Version)); err != nil {
  62. return fmt.Errorf("sync: %v", err)
  63. }
  64. currentVersion := &Version{ID: 1}
  65. has, err := x.Get(currentVersion)
  66. if err != nil {
  67. return fmt.Errorf("get: %v", err)
  68. } else if !has {
  69. // If the version record does not exist we think
  70. // it is a fresh installation and we can skip all migrations.
  71. currentVersion.ID = 0
  72. currentVersion.Version = int64(_MIN_DB_VER + len(migrations))
  73. if _, err = x.InsertOne(currentVersion); err != nil {
  74. return fmt.Errorf("insert: %v", err)
  75. }
  76. }
  77. v := currentVersion.Version
  78. if _MIN_DB_VER > v {
  79. log.Fatal(`
  80. Hi there, thank you for using Gogs for so long!
  81. However, Gogs has stopped supporting auto-migration from your previously installed version.
  82. But the good news is, it's very easy to fix this problem!
  83. You can migrate your older database using a previous release, then you can upgrade to the newest version.
  84. Please save following instructions to somewhere and start working:
  85. - If you were using below 0.6.0 (e.g. 0.5.x), download last supported archive from following link:
  86. https://gogs.io/gogs/releases/tag/v0.7.33
  87. - If you were using below 0.7.0 (e.g. 0.6.x), download last supported archive from following link:
  88. https://gogs.io/gogs/releases/tag/v0.9.141
  89. Once finished downloading,
  90. 1. Extract the archive and to upgrade steps as usual.
  91. 2. Run it once. To verify, you should see some migration traces.
  92. 3. Once it starts web server successfully, stop it.
  93. 4. Now it's time to put back the release archive you originally intent to upgrade.
  94. 5. Enjoy!
  95. In case you're stilling getting this notice, go through instructions again until it disappears.`)
  96. return nil
  97. }
  98. if int(v-_MIN_DB_VER) > len(migrations) {
  99. // User downgraded Gogs.
  100. currentVersion.Version = int64(len(migrations) + _MIN_DB_VER)
  101. _, err = x.Id(1).Update(currentVersion)
  102. return err
  103. }
  104. for i, m := range migrations[v-_MIN_DB_VER:] {
  105. log.Info("Migration: %s", m.Description())
  106. if err = m.Migrate(x); err != nil {
  107. return fmt.Errorf("do migrate: %v", err)
  108. }
  109. currentVersion.Version = v + int64(i) + 1
  110. if _, err = x.Id(1).Update(currentVersion); err != nil {
  111. return err
  112. }
  113. }
  114. return nil
  115. }
  116. func generateOrgRandsAndSalt(x *xorm.Engine) (err error) {
  117. type User struct {
  118. ID int64 `xorm:"pk autoincr"`
  119. Rands string `xorm:"VARCHAR(10)"`
  120. Salt string `xorm:"VARCHAR(10)"`
  121. }
  122. orgs := make([]*User, 0, 10)
  123. if err = x.Where("type=1").And("rands=''").Find(&orgs); err != nil {
  124. return fmt.Errorf("select all organizations: %v", err)
  125. }
  126. sess := x.NewSession()
  127. defer sess.Close()
  128. if err = sess.Begin(); err != nil {
  129. return err
  130. }
  131. for _, org := range orgs {
  132. if org.Rands, err = tool.RandomString(10); err != nil {
  133. return err
  134. }
  135. if org.Salt, err = tool.RandomString(10); err != nil {
  136. return err
  137. }
  138. if _, err = sess.ID(org.ID).Update(org); err != nil {
  139. return err
  140. }
  141. }
  142. return sess.Commit()
  143. }
  144. type TAction struct {
  145. ID int64 `xorm:"pk autoincr"`
  146. CreatedUnix int64
  147. }
  148. func (t *TAction) TableName() string { return "action" }
  149. type TNotice struct {
  150. ID int64 `xorm:"pk autoincr"`
  151. CreatedUnix int64
  152. }
  153. func (t *TNotice) TableName() string { return "notice" }
  154. type TComment struct {
  155. ID int64 `xorm:"pk autoincr"`
  156. CreatedUnix int64
  157. }
  158. func (t *TComment) TableName() string { return "comment" }
  159. type TIssue struct {
  160. ID int64 `xorm:"pk autoincr"`
  161. DeadlineUnix int64
  162. CreatedUnix int64
  163. UpdatedUnix int64
  164. }
  165. func (t *TIssue) TableName() string { return "issue" }
  166. type TMilestone struct {
  167. ID int64 `xorm:"pk autoincr"`
  168. DeadlineUnix int64
  169. ClosedDateUnix int64
  170. }
  171. func (t *TMilestone) TableName() string { return "milestone" }
  172. type TAttachment struct {
  173. ID int64 `xorm:"pk autoincr"`
  174. CreatedUnix int64
  175. }
  176. func (t *TAttachment) TableName() string { return "attachment" }
  177. type TLoginSource struct {
  178. ID int64 `xorm:"pk autoincr"`
  179. CreatedUnix int64
  180. UpdatedUnix int64
  181. }
  182. func (t *TLoginSource) TableName() string { return "login_source" }
  183. type TPull struct {
  184. ID int64 `xorm:"pk autoincr"`
  185. MergedUnix int64
  186. }
  187. func (t *TPull) TableName() string { return "pull_request" }
  188. type TRelease struct {
  189. ID int64 `xorm:"pk autoincr"`
  190. CreatedUnix int64
  191. }
  192. func (t *TRelease) TableName() string { return "release" }
  193. type TRepo struct {
  194. ID int64 `xorm:"pk autoincr"`
  195. CreatedUnix int64
  196. UpdatedUnix int64
  197. }
  198. func (t *TRepo) TableName() string { return "repository" }
  199. type TMirror struct {
  200. ID int64 `xorm:"pk autoincr"`
  201. UpdatedUnix int64
  202. NextUpdateUnix int64
  203. }
  204. func (t *TMirror) TableName() string { return "mirror" }
  205. type TPublicKey struct {
  206. ID int64 `xorm:"pk autoincr"`
  207. CreatedUnix int64
  208. UpdatedUnix int64
  209. }
  210. func (t *TPublicKey) TableName() string { return "public_key" }
  211. type TDeployKey struct {
  212. ID int64 `xorm:"pk autoincr"`
  213. CreatedUnix int64
  214. UpdatedUnix int64
  215. }
  216. func (t *TDeployKey) TableName() string { return "deploy_key" }
  217. type TAccessToken struct {
  218. ID int64 `xorm:"pk autoincr"`
  219. CreatedUnix int64
  220. UpdatedUnix int64
  221. }
  222. func (t *TAccessToken) TableName() string { return "access_token" }
  223. type TUser struct {
  224. ID int64 `xorm:"pk autoincr"`
  225. CreatedUnix int64
  226. UpdatedUnix int64
  227. }
  228. func (t *TUser) TableName() string { return "user" }
  229. type TWebhook struct {
  230. ID int64 `xorm:"pk autoincr"`
  231. CreatedUnix int64
  232. UpdatedUnix int64
  233. }
  234. func (t *TWebhook) TableName() string { return "webhook" }
  235. func convertDateToUnix(x *xorm.Engine) (err error) {
  236. log.Info("This migration could take up to minutes, please be patient.")
  237. type Bean struct {
  238. ID int64 `xorm:"pk autoincr"`
  239. Created time.Time
  240. Updated time.Time
  241. Merged time.Time
  242. Deadline time.Time
  243. ClosedDate time.Time
  244. NextUpdate time.Time
  245. }
  246. var tables = []struct {
  247. name string
  248. cols []string
  249. bean interface{}
  250. }{
  251. {"action", []string{"created"}, new(TAction)},
  252. {"notice", []string{"created"}, new(TNotice)},
  253. {"comment", []string{"created"}, new(TComment)},
  254. {"issue", []string{"deadline", "created", "updated"}, new(TIssue)},
  255. {"milestone", []string{"deadline", "closed_date"}, new(TMilestone)},
  256. {"attachment", []string{"created"}, new(TAttachment)},
  257. {"login_source", []string{"created", "updated"}, new(TLoginSource)},
  258. {"pull_request", []string{"merged"}, new(TPull)},
  259. {"release", []string{"created"}, new(TRelease)},
  260. {"repository", []string{"created", "updated"}, new(TRepo)},
  261. {"mirror", []string{"updated", "next_update"}, new(TMirror)},
  262. {"public_key", []string{"created", "updated"}, new(TPublicKey)},
  263. {"deploy_key", []string{"created", "updated"}, new(TDeployKey)},
  264. {"access_token", []string{"created", "updated"}, new(TAccessToken)},
  265. {"user", []string{"created", "updated"}, new(TUser)},
  266. {"webhook", []string{"created", "updated"}, new(TWebhook)},
  267. }
  268. for _, table := range tables {
  269. log.Info("Converting table: %s", table.name)
  270. if err = x.Sync2(table.bean); err != nil {
  271. return fmt.Errorf("Sync [table: %s]: %v", table.name, err)
  272. }
  273. offset := 0
  274. for {
  275. beans := make([]*Bean, 0, 100)
  276. if err = x.SQL(fmt.Sprintf("SELECT * FROM `%s` ORDER BY id ASC LIMIT 100 OFFSET %d",
  277. table.name, offset)).Find(&beans); err != nil {
  278. return fmt.Errorf("select beans [table: %s, offset: %d]: %v", table.name, offset, err)
  279. }
  280. log.Trace("Table [%s]: offset: %d, beans: %d", table.name, offset, len(beans))
  281. if len(beans) == 0 {
  282. break
  283. }
  284. offset += 100
  285. baseSQL := "UPDATE `" + table.name + "` SET "
  286. for _, bean := range beans {
  287. valSQLs := make([]string, 0, len(table.cols))
  288. for _, col := range table.cols {
  289. fieldSQL := ""
  290. fieldSQL += col + "_unix = "
  291. switch col {
  292. case "deadline":
  293. if bean.Deadline.IsZero() {
  294. continue
  295. }
  296. fieldSQL += com.ToStr(bean.Deadline.Unix())
  297. case "created":
  298. fieldSQL += com.ToStr(bean.Created.Unix())
  299. case "updated":
  300. fieldSQL += com.ToStr(bean.Updated.Unix())
  301. case "closed_date":
  302. fieldSQL += com.ToStr(bean.ClosedDate.Unix())
  303. case "merged":
  304. fieldSQL += com.ToStr(bean.Merged.Unix())
  305. case "next_update":
  306. fieldSQL += com.ToStr(bean.NextUpdate.Unix())
  307. }
  308. valSQLs = append(valSQLs, fieldSQL)
  309. }
  310. if len(valSQLs) == 0 {
  311. continue
  312. }
  313. if _, err = x.Exec(baseSQL + strings.Join(valSQLs, ",") + " WHERE id = " + com.ToStr(bean.ID)); err != nil {
  314. return fmt.Errorf("update bean [table: %s, id: %d]: %v", table.name, bean.ID, err)
  315. }
  316. }
  317. }
  318. }
  319. return nil
  320. }