mirror.go 7.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269
  1. // Copyright 2016 The Gogs Authors. All rights reserved.
  2. // Use of this source code is governed by a MIT-style
  3. // license that can be found in the LICENSE file.
  4. package models
  5. import (
  6. "fmt"
  7. "strings"
  8. "time"
  9. "github.com/Unknwon/com"
  10. "github.com/go-xorm/xorm"
  11. log "gopkg.in/clog.v1"
  12. "gopkg.in/ini.v1"
  13. "github.com/gogits/git-module"
  14. "github.com/gogits/gogs/modules/process"
  15. "github.com/gogits/gogs/modules/setting"
  16. "github.com/gogits/gogs/modules/sync"
  17. )
  18. var MirrorQueue = sync.NewUniqueQueue(setting.Repository.MirrorQueueLength)
  19. // Mirror represents mirror information of a repository.
  20. type Mirror struct {
  21. ID int64 `xorm:"pk autoincr"`
  22. RepoID int64
  23. Repo *Repository `xorm:"-"`
  24. Interval int // Hour.
  25. EnablePrune bool `xorm:"NOT NULL DEFAULT true"`
  26. Updated time.Time `xorm:"-"`
  27. UpdatedUnix int64
  28. NextUpdate time.Time `xorm:"-"`
  29. NextUpdateUnix int64
  30. address string `xorm:"-"`
  31. }
  32. func (m *Mirror) BeforeInsert() {
  33. m.UpdatedUnix = time.Now().Unix()
  34. m.NextUpdateUnix = m.NextUpdate.Unix()
  35. }
  36. func (m *Mirror) BeforeUpdate() {
  37. m.UpdatedUnix = time.Now().Unix()
  38. m.NextUpdateUnix = m.NextUpdate.Unix()
  39. }
  40. func (m *Mirror) AfterSet(colName string, _ xorm.Cell) {
  41. var err error
  42. switch colName {
  43. case "repo_id":
  44. m.Repo, err = GetRepositoryByID(m.RepoID)
  45. if err != nil {
  46. log.Error(3, "GetRepositoryByID [%d]: %v", m.ID, err)
  47. }
  48. case "updated_unix":
  49. m.Updated = time.Unix(m.UpdatedUnix, 0).Local()
  50. case "next_updated_unix":
  51. m.NextUpdate = time.Unix(m.NextUpdateUnix, 0).Local()
  52. }
  53. }
  54. // ScheduleNextUpdate calculates and sets next update time.
  55. func (m *Mirror) ScheduleNextUpdate() {
  56. m.NextUpdate = time.Now().Add(time.Duration(m.Interval) * time.Hour)
  57. }
  58. func (m *Mirror) readAddress() {
  59. if len(m.address) > 0 {
  60. return
  61. }
  62. cfg, err := ini.Load(m.Repo.GitConfigPath())
  63. if err != nil {
  64. log.Error(2, "Load: %v", err)
  65. return
  66. }
  67. m.address = cfg.Section("remote \"origin\"").Key("url").Value()
  68. }
  69. // HandleCloneUserCredentials replaces user credentials from HTTP/HTTPS URL
  70. // with placeholder <credentials>.
  71. // It will fail for any other forms of clone addresses.
  72. func HandleCloneUserCredentials(url string, mosaics bool) string {
  73. i := strings.Index(url, "@")
  74. if i == -1 {
  75. return url
  76. }
  77. start := strings.Index(url, "://")
  78. if start == -1 {
  79. return url
  80. }
  81. if mosaics {
  82. return url[:start+3] + "<credentials>" + url[i:]
  83. }
  84. return url[:start+3] + url[i+1:]
  85. }
  86. // Address returns mirror address from Git repository config without credentials.
  87. func (m *Mirror) Address() string {
  88. m.readAddress()
  89. return HandleCloneUserCredentials(m.address, false)
  90. }
  91. // MosaicsAddress returns mirror address from Git repository config with credentials under mosaics.
  92. func (m *Mirror) MosaicsAddress() string {
  93. m.readAddress()
  94. return HandleCloneUserCredentials(m.address, true)
  95. }
  96. // FullAddress returns mirror address from Git repository config.
  97. func (m *Mirror) FullAddress() string {
  98. m.readAddress()
  99. return m.address
  100. }
  101. // SaveAddress writes new address to Git repository config.
  102. func (m *Mirror) SaveAddress(addr string) error {
  103. configPath := m.Repo.GitConfigPath()
  104. cfg, err := ini.Load(configPath)
  105. if err != nil {
  106. return fmt.Errorf("Load: %v", err)
  107. }
  108. cfg.Section("remote \"origin\"").Key("url").SetValue(addr)
  109. return cfg.SaveToIndent(configPath, "\t")
  110. }
  111. // runSync returns true if sync finished without error.
  112. func (m *Mirror) runSync() bool {
  113. repoPath := m.Repo.RepoPath()
  114. wikiPath := m.Repo.WikiPath()
  115. timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
  116. // Do a fast-fail testing against on repository URL to ensure it is accessible under
  117. // good condition to prevent long blocking on URL resolution without syncing anything.
  118. if !git.IsRepoURLAccessible(git.NetworkOptions{
  119. URL: m.FullAddress(),
  120. Timeout: 10 * time.Second,
  121. }) {
  122. desc := fmt.Sprintf("Source URL of mirror repository '%s' is not accessible: %s", m.Repo.FullName(), m.MosaicsAddress())
  123. if err := CreateRepositoryNotice(desc); err != nil {
  124. log.Error(2, "CreateRepositoryNotice: %v", err)
  125. }
  126. return false
  127. }
  128. gitArgs := []string{"remote", "update"}
  129. if m.EnablePrune {
  130. gitArgs = append(gitArgs, "--prune")
  131. }
  132. if _, stderr, err := process.ExecDir(
  133. timeout, repoPath, fmt.Sprintf("Mirror.runSync: %s", repoPath),
  134. "git", gitArgs...); err != nil {
  135. desc := fmt.Sprintf("Fail to update mirror repository '%s': %s", repoPath, stderr)
  136. log.Error(2, desc)
  137. if err = CreateRepositoryNotice(desc); err != nil {
  138. log.Error(2, "CreateRepositoryNotice: %v", err)
  139. }
  140. return false
  141. }
  142. if m.Repo.HasWiki() {
  143. if _, stderr, err := process.ExecDir(
  144. timeout, wikiPath, fmt.Sprintf("Mirror.runSync: %s", wikiPath),
  145. "git", "remote", "update", "--prune"); err != nil {
  146. desc := fmt.Sprintf("Fail to update mirror wiki repository '%s': %s", wikiPath, stderr)
  147. log.Error(2, desc)
  148. if err = CreateRepositoryNotice(desc); err != nil {
  149. log.Error(2, "CreateRepositoryNotice: %v", err)
  150. }
  151. return false
  152. }
  153. }
  154. return true
  155. }
  156. func getMirrorByRepoID(e Engine, repoID int64) (*Mirror, error) {
  157. m := &Mirror{RepoID: repoID}
  158. has, err := e.Get(m)
  159. if err != nil {
  160. return nil, err
  161. } else if !has {
  162. return nil, ErrMirrorNotExist
  163. }
  164. return m, nil
  165. }
  166. // GetMirrorByRepoID returns mirror information of a repository.
  167. func GetMirrorByRepoID(repoID int64) (*Mirror, error) {
  168. return getMirrorByRepoID(x, repoID)
  169. }
  170. func updateMirror(e Engine, m *Mirror) error {
  171. _, err := e.Id(m.ID).AllCols().Update(m)
  172. return err
  173. }
  174. func UpdateMirror(m *Mirror) error {
  175. return updateMirror(x, m)
  176. }
  177. func DeleteMirrorByRepoID(repoID int64) error {
  178. _, err := x.Delete(&Mirror{RepoID: repoID})
  179. return err
  180. }
  181. // MirrorUpdate checks and updates mirror repositories.
  182. func MirrorUpdate() {
  183. if taskStatusTable.IsRunning(_MIRROR_UPDATE) {
  184. return
  185. }
  186. taskStatusTable.Start(_MIRROR_UPDATE)
  187. defer taskStatusTable.Stop(_MIRROR_UPDATE)
  188. log.Trace("Doing: MirrorUpdate")
  189. if err := x.Where("next_update_unix<=?", time.Now().Unix()).Iterate(new(Mirror), func(idx int, bean interface{}) error {
  190. m := bean.(*Mirror)
  191. if m.Repo == nil {
  192. log.Error(2, "Disconnected mirror repository found: %d", m.ID)
  193. return nil
  194. }
  195. MirrorQueue.Add(m.RepoID)
  196. return nil
  197. }); err != nil {
  198. log.Error(2, "MirrorUpdate: %v", err)
  199. }
  200. }
  201. // SyncMirrors checks and syncs mirrors.
  202. // TODO: sync more mirrors at same time.
  203. func SyncMirrors() {
  204. // Start listening on new sync requests.
  205. for repoID := range MirrorQueue.Queue() {
  206. log.Trace("SyncMirrors [repo_id: %v]", repoID)
  207. MirrorQueue.Remove(repoID)
  208. m, err := GetMirrorByRepoID(com.StrTo(repoID).MustInt64())
  209. if err != nil {
  210. log.Error(2, "GetMirrorByRepoID [%s]: %v", m.RepoID, err)
  211. continue
  212. }
  213. if !m.runSync() {
  214. continue
  215. }
  216. m.ScheduleNextUpdate()
  217. if err = UpdateMirror(m); err != nil {
  218. log.Error(2, "UpdateMirror [%s]: %v", m.RepoID, err)
  219. continue
  220. }
  221. // Update repository last updated time
  222. if _, err = x.Exec("UPDATE repository SET updated_unix = ? WHERE id = ?", time.Now().Unix(), m.RepoID); err != nil {
  223. log.Error(2, "Update repository 'updated_unix' [%s]: %v", m.RepoID, err)
  224. }
  225. }
  226. }
  227. func InitSyncMirrors() {
  228. go SyncMirrors()
  229. }