repo_mirror.go 6.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243
  1. // Copyright 2016 The Gogs Authors. All rights reserved.
  2. // Use of this source code is governed by a MIT-style
  3. // license that can be found in the LICENSE file.
  4. package models
  5. import (
  6. "fmt"
  7. "strings"
  8. "time"
  9. "github.com/Unknwon/com"
  10. "github.com/go-xorm/xorm"
  11. "gopkg.in/ini.v1"
  12. "github.com/gogits/gogs/modules/log"
  13. "github.com/gogits/gogs/modules/process"
  14. "github.com/gogits/gogs/modules/setting"
  15. "github.com/gogits/gogs/modules/sync"
  16. )
  17. var MirrorQueue = sync.NewUniqueQueue(setting.Repository.MirrorQueueLength)
  18. // Mirror represents mirror information of a repository.
  19. type Mirror struct {
  20. ID int64 `xorm:"pk autoincr"`
  21. RepoID int64
  22. Repo *Repository `xorm:"-"`
  23. Interval int // Hour.
  24. EnablePrune bool `xorm:"NOT NULL DEFAULT true"`
  25. Updated time.Time `xorm:"-"`
  26. UpdatedUnix int64
  27. NextUpdate time.Time `xorm:"-"`
  28. NextUpdateUnix int64
  29. address string `xorm:"-"`
  30. }
  31. func (m *Mirror) BeforeInsert() {
  32. m.NextUpdateUnix = m.NextUpdate.Unix()
  33. }
  34. func (m *Mirror) BeforeUpdate() {
  35. m.UpdatedUnix = time.Now().Unix()
  36. m.NextUpdateUnix = m.NextUpdate.Unix()
  37. }
  38. func (m *Mirror) AfterSet(colName string, _ xorm.Cell) {
  39. var err error
  40. switch colName {
  41. case "repo_id":
  42. m.Repo, err = GetRepositoryByID(m.RepoID)
  43. if err != nil {
  44. log.Error(3, "GetRepositoryByID[%d]: %v", m.ID, err)
  45. }
  46. case "updated_unix":
  47. m.Updated = time.Unix(m.UpdatedUnix, 0).Local()
  48. case "next_updated_unix":
  49. m.NextUpdate = time.Unix(m.NextUpdateUnix, 0).Local()
  50. }
  51. }
  52. // ScheduleNextUpdate calculates and sets next update time.
  53. func (m *Mirror) ScheduleNextUpdate() {
  54. m.NextUpdate = time.Now().Add(time.Duration(m.Interval) * time.Hour)
  55. }
  56. func (m *Mirror) readAddress() {
  57. if len(m.address) > 0 {
  58. return
  59. }
  60. cfg, err := ini.Load(m.Repo.GitConfigPath())
  61. if err != nil {
  62. log.Error(4, "Load: %v", err)
  63. return
  64. }
  65. m.address = cfg.Section("remote \"origin\"").Key("url").Value()
  66. }
  67. // HandleCloneUserCredentials replaces user credentials from HTTP/HTTPS URL
  68. // with placeholder <credentials>.
  69. // It will fail for any other forms of clone addresses.
  70. func HandleCloneUserCredentials(url string, mosaics bool) string {
  71. i := strings.Index(url, "@")
  72. if i == -1 {
  73. return url
  74. }
  75. start := strings.Index(url, "://")
  76. if start == -1 {
  77. return url
  78. }
  79. if mosaics {
  80. return url[:start+3] + "<credentials>" + url[i:]
  81. }
  82. return url[:start+3] + url[i+1:]
  83. }
  84. // Address returns mirror address from Git repository config without credentials.
  85. func (m *Mirror) Address() string {
  86. m.readAddress()
  87. return HandleCloneUserCredentials(m.address, false)
  88. }
  89. // FullAddress returns mirror address from Git repository config.
  90. func (m *Mirror) FullAddress() string {
  91. m.readAddress()
  92. return m.address
  93. }
  94. // SaveAddress writes new address to Git repository config.
  95. func (m *Mirror) SaveAddress(addr string) error {
  96. configPath := m.Repo.GitConfigPath()
  97. cfg, err := ini.Load(configPath)
  98. if err != nil {
  99. return fmt.Errorf("Load: %v", err)
  100. }
  101. cfg.Section("remote \"origin\"").Key("url").SetValue(addr)
  102. return cfg.SaveToIndent(configPath, "\t")
  103. }
  104. // runSync returns true if sync finished without error.
  105. func (m *Mirror) runSync() bool {
  106. repoPath := m.Repo.RepoPath()
  107. wikiPath := m.Repo.WikiPath()
  108. timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
  109. gitArgs := []string{"remote", "update"}
  110. if m.EnablePrune {
  111. gitArgs = append(gitArgs, "--prune")
  112. }
  113. if _, stderr, err := process.ExecDir(
  114. timeout, repoPath, fmt.Sprintf("Mirror.runSync: %s", repoPath),
  115. "git", gitArgs...); err != nil {
  116. desc := fmt.Sprintf("Fail to update mirror repository '%s': %s", repoPath, stderr)
  117. log.Error(4, desc)
  118. if err = CreateRepositoryNotice(desc); err != nil {
  119. log.Error(4, "CreateRepositoryNotice: %v", err)
  120. }
  121. return false
  122. }
  123. if m.Repo.HasWiki() {
  124. if _, stderr, err := process.ExecDir(
  125. timeout, wikiPath, fmt.Sprintf("Mirror.runSync: %s", wikiPath),
  126. "git", "remote", "update", "--prune"); err != nil {
  127. desc := fmt.Sprintf("Fail to update mirror wiki repository '%s': %s", wikiPath, stderr)
  128. log.Error(4, desc)
  129. if err = CreateRepositoryNotice(desc); err != nil {
  130. log.Error(4, "CreateRepositoryNotice: %v", err)
  131. }
  132. return false
  133. }
  134. }
  135. return true
  136. }
  137. func getMirrorByRepoID(e Engine, repoID int64) (*Mirror, error) {
  138. m := &Mirror{RepoID: repoID}
  139. has, err := e.Get(m)
  140. if err != nil {
  141. return nil, err
  142. } else if !has {
  143. return nil, ErrMirrorNotExist
  144. }
  145. return m, nil
  146. }
  147. // GetMirrorByRepoID returns mirror information of a repository.
  148. func GetMirrorByRepoID(repoID int64) (*Mirror, error) {
  149. return getMirrorByRepoID(x, repoID)
  150. }
  151. func updateMirror(e Engine, m *Mirror) error {
  152. _, err := e.Id(m.ID).AllCols().Update(m)
  153. return err
  154. }
  155. func UpdateMirror(m *Mirror) error {
  156. return updateMirror(x, m)
  157. }
  158. func DeleteMirrorByRepoID(repoID int64) error {
  159. _, err := x.Delete(&Mirror{RepoID: repoID})
  160. return err
  161. }
  162. // MirrorUpdate checks and updates mirror repositories.
  163. func MirrorUpdate() {
  164. if taskStatusTable.IsRunning(_MIRROR_UPDATE) {
  165. return
  166. }
  167. taskStatusTable.Start(_MIRROR_UPDATE)
  168. defer taskStatusTable.Stop(_MIRROR_UPDATE)
  169. log.Trace("Doing: MirrorUpdate")
  170. if err := x.Where("next_update_unix<=?", time.Now().Unix()).Iterate(new(Mirror), func(idx int, bean interface{}) error {
  171. m := bean.(*Mirror)
  172. if m.Repo == nil {
  173. log.Error(4, "Disconnected mirror repository found: %d", m.ID)
  174. return nil
  175. }
  176. MirrorQueue.Add(m.RepoID)
  177. return nil
  178. }); err != nil {
  179. log.Error(4, "MirrorUpdate: %v", err)
  180. }
  181. }
  182. // SyncMirrors checks and syncs mirrors.
  183. // TODO: sync more mirrors at same time.
  184. func SyncMirrors() {
  185. // Start listening on new sync requests.
  186. for repoID := range MirrorQueue.Queue() {
  187. log.Trace("SyncMirrors [repo_id: %v]", repoID)
  188. MirrorQueue.Remove(repoID)
  189. m, err := GetMirrorByRepoID(com.StrTo(repoID).MustInt64())
  190. if err != nil {
  191. log.Error(4, "GetMirrorByRepoID [%d]: %v", repoID, err)
  192. continue
  193. }
  194. if !m.runSync() {
  195. continue
  196. }
  197. m.ScheduleNextUpdate()
  198. if err = UpdateMirror(m); err != nil {
  199. log.Error(4, "UpdateMirror [%d]: %v", repoID, err)
  200. continue
  201. }
  202. }
  203. }
  204. func InitSyncMirrors() {
  205. go SyncMirrors()
  206. }