repo_mirror.go 6.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244
  1. // Copyright 2016 The Gogs Authors. All rights reserved.
  2. // Use of this source code is governed by a MIT-style
  3. // license that can be found in the LICENSE file.
  4. package models
  5. import (
  6. "fmt"
  7. "strings"
  8. "time"
  9. "github.com/Unknwon/com"
  10. "github.com/go-xorm/xorm"
  11. "gopkg.in/ini.v1"
  12. "github.com/gogits/gogs/modules/log"
  13. "github.com/gogits/gogs/modules/process"
  14. "github.com/gogits/gogs/modules/setting"
  15. "github.com/gogits/gogs/modules/sync"
  16. )
  17. var MirrorQueue = sync.NewUniqueQueue(setting.Repository.MirrorQueueLength)
  18. // Mirror represents mirror information of a repository.
  19. type Mirror struct {
  20. ID int64 `xorm:"pk autoincr"`
  21. RepoID int64
  22. Repo *Repository `xorm:"-"`
  23. Interval int // Hour.
  24. EnablePrune bool `xorm:"NOT NULL DEFAULT true"`
  25. Updated time.Time `xorm:"-"`
  26. UpdatedUnix int64
  27. NextUpdate time.Time `xorm:"-"`
  28. NextUpdateUnix int64
  29. address string `xorm:"-"`
  30. }
  31. func (m *Mirror) BeforeInsert() {
  32. m.UpdatedUnix = time.Now().Unix()
  33. m.NextUpdateUnix = m.NextUpdate.Unix()
  34. }
  35. func (m *Mirror) BeforeUpdate() {
  36. m.UpdatedUnix = time.Now().Unix()
  37. m.NextUpdateUnix = m.NextUpdate.Unix()
  38. }
  39. func (m *Mirror) AfterSet(colName string, _ xorm.Cell) {
  40. var err error
  41. switch colName {
  42. case "repo_id":
  43. m.Repo, err = GetRepositoryByID(m.RepoID)
  44. if err != nil {
  45. log.Error(3, "GetRepositoryByID[%d]: %v", m.ID, err)
  46. }
  47. case "updated_unix":
  48. m.Updated = time.Unix(m.UpdatedUnix, 0).Local()
  49. case "next_updated_unix":
  50. m.NextUpdate = time.Unix(m.NextUpdateUnix, 0).Local()
  51. }
  52. }
  53. // ScheduleNextUpdate calculates and sets next update time.
  54. func (m *Mirror) ScheduleNextUpdate() {
  55. m.NextUpdate = time.Now().Add(time.Duration(m.Interval) * time.Hour)
  56. }
  57. func (m *Mirror) readAddress() {
  58. if len(m.address) > 0 {
  59. return
  60. }
  61. cfg, err := ini.Load(m.Repo.GitConfigPath())
  62. if err != nil {
  63. log.Error(4, "Load: %v", err)
  64. return
  65. }
  66. m.address = cfg.Section("remote \"origin\"").Key("url").Value()
  67. }
  68. // HandleCloneUserCredentials replaces user credentials from HTTP/HTTPS URL
  69. // with placeholder <credentials>.
  70. // It will fail for any other forms of clone addresses.
  71. func HandleCloneUserCredentials(url string, mosaics bool) string {
  72. i := strings.Index(url, "@")
  73. if i == -1 {
  74. return url
  75. }
  76. start := strings.Index(url, "://")
  77. if start == -1 {
  78. return url
  79. }
  80. if mosaics {
  81. return url[:start+3] + "<credentials>" + url[i:]
  82. }
  83. return url[:start+3] + url[i+1:]
  84. }
  85. // Address returns mirror address from Git repository config without credentials.
  86. func (m *Mirror) Address() string {
  87. m.readAddress()
  88. return HandleCloneUserCredentials(m.address, false)
  89. }
  90. // FullAddress returns mirror address from Git repository config.
  91. func (m *Mirror) FullAddress() string {
  92. m.readAddress()
  93. return m.address
  94. }
  95. // SaveAddress writes new address to Git repository config.
  96. func (m *Mirror) SaveAddress(addr string) error {
  97. configPath := m.Repo.GitConfigPath()
  98. cfg, err := ini.Load(configPath)
  99. if err != nil {
  100. return fmt.Errorf("Load: %v", err)
  101. }
  102. cfg.Section("remote \"origin\"").Key("url").SetValue(addr)
  103. return cfg.SaveToIndent(configPath, "\t")
  104. }
  105. // runSync returns true if sync finished without error.
  106. func (m *Mirror) runSync() bool {
  107. repoPath := m.Repo.RepoPath()
  108. wikiPath := m.Repo.WikiPath()
  109. timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
  110. gitArgs := []string{"remote", "update"}
  111. if m.EnablePrune {
  112. gitArgs = append(gitArgs, "--prune")
  113. }
  114. if _, stderr, err := process.ExecDir(
  115. timeout, repoPath, fmt.Sprintf("Mirror.runSync: %s", repoPath),
  116. "git", gitArgs...); err != nil {
  117. desc := fmt.Sprintf("Fail to update mirror repository '%s': %s", repoPath, stderr)
  118. log.Error(4, desc)
  119. if err = CreateRepositoryNotice(desc); err != nil {
  120. log.Error(4, "CreateRepositoryNotice: %v", err)
  121. }
  122. return false
  123. }
  124. if m.Repo.HasWiki() {
  125. if _, stderr, err := process.ExecDir(
  126. timeout, wikiPath, fmt.Sprintf("Mirror.runSync: %s", wikiPath),
  127. "git", "remote", "update", "--prune"); err != nil {
  128. desc := fmt.Sprintf("Fail to update mirror wiki repository '%s': %s", wikiPath, stderr)
  129. log.Error(4, desc)
  130. if err = CreateRepositoryNotice(desc); err != nil {
  131. log.Error(4, "CreateRepositoryNotice: %v", err)
  132. }
  133. return false
  134. }
  135. }
  136. return true
  137. }
  138. func getMirrorByRepoID(e Engine, repoID int64) (*Mirror, error) {
  139. m := &Mirror{RepoID: repoID}
  140. has, err := e.Get(m)
  141. if err != nil {
  142. return nil, err
  143. } else if !has {
  144. return nil, ErrMirrorNotExist
  145. }
  146. return m, nil
  147. }
  148. // GetMirrorByRepoID returns mirror information of a repository.
  149. func GetMirrorByRepoID(repoID int64) (*Mirror, error) {
  150. return getMirrorByRepoID(x, repoID)
  151. }
  152. func updateMirror(e Engine, m *Mirror) error {
  153. _, err := e.Id(m.ID).AllCols().Update(m)
  154. return err
  155. }
  156. func UpdateMirror(m *Mirror) error {
  157. return updateMirror(x, m)
  158. }
  159. func DeleteMirrorByRepoID(repoID int64) error {
  160. _, err := x.Delete(&Mirror{RepoID: repoID})
  161. return err
  162. }
  163. // MirrorUpdate checks and updates mirror repositories.
  164. func MirrorUpdate() {
  165. if taskStatusTable.IsRunning(_MIRROR_UPDATE) {
  166. return
  167. }
  168. taskStatusTable.Start(_MIRROR_UPDATE)
  169. defer taskStatusTable.Stop(_MIRROR_UPDATE)
  170. log.Trace("Doing: MirrorUpdate")
  171. if err := x.Where("next_update_unix<=?", time.Now().Unix()).Iterate(new(Mirror), func(idx int, bean interface{}) error {
  172. m := bean.(*Mirror)
  173. if m.Repo == nil {
  174. log.Error(4, "Disconnected mirror repository found: %d", m.ID)
  175. return nil
  176. }
  177. MirrorQueue.Add(m.RepoID)
  178. return nil
  179. }); err != nil {
  180. log.Error(4, "MirrorUpdate: %v", err)
  181. }
  182. }
  183. // SyncMirrors checks and syncs mirrors.
  184. // TODO: sync more mirrors at same time.
  185. func SyncMirrors() {
  186. // Start listening on new sync requests.
  187. for repoID := range MirrorQueue.Queue() {
  188. log.Trace("SyncMirrors [repo_id: %v]", repoID)
  189. MirrorQueue.Remove(repoID)
  190. m, err := GetMirrorByRepoID(com.StrTo(repoID).MustInt64())
  191. if err != nil {
  192. log.Error(4, "GetMirrorByRepoID [%s]: %v", repoID, err)
  193. continue
  194. }
  195. if !m.runSync() {
  196. continue
  197. }
  198. m.ScheduleNextUpdate()
  199. if err = UpdateMirror(m); err != nil {
  200. log.Error(4, "UpdateMirror [%s]: %v", repoID, err)
  201. continue
  202. }
  203. }
  204. }
  205. func InitSyncMirrors() {
  206. go SyncMirrors()
  207. }