models.go 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349
  1. // Copyright 2014 The Gogs Authors. All rights reserved.
  2. // Use of this source code is governed by a MIT-style
  3. // license that can be found in the LICENSE file.
  4. package db
  5. import (
  6. "bufio"
  7. "database/sql"
  8. "fmt"
  9. "net/url"
  10. "os"
  11. "path"
  12. "strings"
  13. "time"
  14. "github.com/json-iterator/go"
  15. "github.com/unknwon/com"
  16. log "unknwon.dev/clog/v2"
  17. "xorm.io/core"
  18. "xorm.io/xorm"
  19. "gogs.io/gogs/internal/conf"
  20. "gogs.io/gogs/internal/db/migrations"
  21. )
  22. // Engine represents a XORM engine or session.
  23. type Engine interface {
  24. Delete(interface{}) (int64, error)
  25. Exec(...interface{}) (sql.Result, error)
  26. Find(interface{}, ...interface{}) error
  27. Get(interface{}) (bool, error)
  28. ID(interface{}) *xorm.Session
  29. In(string, ...interface{}) *xorm.Session
  30. Insert(...interface{}) (int64, error)
  31. InsertOne(interface{}) (int64, error)
  32. Iterate(interface{}, xorm.IterFunc) error
  33. Sql(string, ...interface{}) *xorm.Session
  34. Table(interface{}) *xorm.Session
  35. Where(interface{}, ...interface{}) *xorm.Session
  36. }
  37. var (
  38. x *xorm.Engine
  39. legacyTables []interface{}
  40. HasEngine bool
  41. )
  42. func init() {
  43. legacyTables = append(legacyTables,
  44. new(User), new(PublicKey), new(TwoFactor), new(TwoFactorRecoveryCode),
  45. new(Repository), new(DeployKey), new(Collaboration), new(Access), new(Upload),
  46. new(Watch), new(Star), new(Follow), new(Action),
  47. new(Issue), new(PullRequest), new(Comment), new(Attachment), new(IssueUser),
  48. new(Label), new(IssueLabel), new(Milestone),
  49. new(Mirror), new(Release), new(LoginSource), new(Webhook), new(HookTask),
  50. new(ProtectBranch), new(ProtectBranchWhitelist),
  51. new(Team), new(OrgUser), new(TeamUser), new(TeamRepo),
  52. new(Notice), new(EmailAddress))
  53. gonicNames := []string{"SSL"}
  54. for _, name := range gonicNames {
  55. core.LintGonicMapper[name] = true
  56. }
  57. }
  58. func getEngine() (*xorm.Engine, error) {
  59. Param := "?"
  60. if strings.Contains(conf.Database.Name, Param) {
  61. Param = "&"
  62. }
  63. connStr := ""
  64. switch conf.Database.Type {
  65. case "mysql":
  66. conf.UseMySQL = true
  67. if conf.Database.Host[0] == '/' { // looks like a unix socket
  68. connStr = fmt.Sprintf("%s:%s@unix(%s)/%s%scharset=utf8mb4&parseTime=true",
  69. conf.Database.User, conf.Database.Password, conf.Database.Host, conf.Database.Name, Param)
  70. } else {
  71. connStr = fmt.Sprintf("%s:%s@tcp(%s)/%s%scharset=utf8mb4&parseTime=true",
  72. conf.Database.User, conf.Database.Password, conf.Database.Host, conf.Database.Name, Param)
  73. }
  74. var engineParams = map[string]string{"rowFormat": "DYNAMIC"}
  75. return xorm.NewEngineWithParams(conf.Database.Type, connStr, engineParams)
  76. case "postgres":
  77. conf.UsePostgreSQL = true
  78. host, port := parsePostgreSQLHostPort(conf.Database.Host)
  79. if host[0] == '/' { // looks like a unix socket
  80. connStr = fmt.Sprintf("postgres://%s:%s@:%s/%s%ssslmode=%s&host=%s",
  81. url.QueryEscape(conf.Database.User), url.QueryEscape(conf.Database.Password), port, conf.Database.Name, Param, conf.Database.SSLMode, host)
  82. } else {
  83. connStr = fmt.Sprintf("postgres://%s:%s@%s:%s/%s%ssslmode=%s",
  84. url.QueryEscape(conf.Database.User), url.QueryEscape(conf.Database.Password), host, port, conf.Database.Name, Param, conf.Database.SSLMode)
  85. }
  86. case "mssql":
  87. conf.UseMSSQL = true
  88. host, port := parseMSSQLHostPort(conf.Database.Host)
  89. connStr = fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", host, port, conf.Database.Name, conf.Database.User, conf.Database.Password)
  90. case "sqlite3":
  91. if err := os.MkdirAll(path.Dir(conf.Database.Path), os.ModePerm); err != nil {
  92. return nil, fmt.Errorf("create directories: %v", err)
  93. }
  94. conf.UseSQLite3 = true
  95. connStr = "file:" + conf.Database.Path + "?cache=shared&mode=rwc"
  96. default:
  97. return nil, fmt.Errorf("unknown database type: %s", conf.Database.Type)
  98. }
  99. return xorm.NewEngine(conf.Database.Type, connStr)
  100. }
  101. func NewTestEngine() error {
  102. x, err := getEngine()
  103. if err != nil {
  104. return fmt.Errorf("connect to database: %v", err)
  105. }
  106. x.SetMapper(core.GonicMapper{})
  107. return x.StoreEngine("InnoDB").Sync2(legacyTables...)
  108. }
  109. func SetEngine() (err error) {
  110. x, err = getEngine()
  111. if err != nil {
  112. return fmt.Errorf("connect to database: %v", err)
  113. }
  114. x.SetMapper(core.GonicMapper{})
  115. // WARNING: for serv command, MUST remove the output to os.stdout,
  116. // so use log file to instead print to stdout.
  117. sec := conf.File.Section("log.xorm")
  118. logger, err := log.NewFileWriter(path.Join(conf.Log.RootPath, "xorm.log"),
  119. log.FileRotationConfig{
  120. Rotate: sec.Key("ROTATE").MustBool(true),
  121. Daily: sec.Key("ROTATE_DAILY").MustBool(true),
  122. MaxSize: sec.Key("MAX_SIZE").MustInt64(100) * 1024 * 1024,
  123. MaxDays: sec.Key("MAX_DAYS").MustInt64(3),
  124. })
  125. if err != nil {
  126. return fmt.Errorf("create 'xorm.log': %v", err)
  127. }
  128. x.SetMaxOpenConns(conf.Database.MaxOpenConns)
  129. x.SetMaxIdleConns(conf.Database.MaxIdleConns)
  130. x.SetConnMaxLifetime(time.Second)
  131. if conf.IsProdMode() {
  132. x.SetLogger(xorm.NewSimpleLogger3(logger, xorm.DEFAULT_LOG_PREFIX, xorm.DEFAULT_LOG_FLAG, core.LOG_WARNING))
  133. } else {
  134. x.SetLogger(xorm.NewSimpleLogger(logger))
  135. }
  136. x.ShowSQL(true)
  137. return Init()
  138. }
  139. func NewEngine() (err error) {
  140. if err = SetEngine(); err != nil {
  141. return err
  142. }
  143. if err = migrations.Migrate(x); err != nil {
  144. return fmt.Errorf("migrate: %v", err)
  145. }
  146. if err = x.StoreEngine("InnoDB").Sync2(legacyTables...); err != nil {
  147. return fmt.Errorf("sync structs to database tables: %v\n", err)
  148. }
  149. return nil
  150. }
  151. type Statistic struct {
  152. Counter struct {
  153. User, Org, PublicKey,
  154. Repo, Watch, Star, Action, Access,
  155. Issue, Comment, Oauth, Follow,
  156. Mirror, Release, LoginSource, Webhook,
  157. Milestone, Label, HookTask,
  158. Team, UpdateTask, Attachment int64
  159. }
  160. }
  161. func GetStatistic() (stats Statistic) {
  162. stats.Counter.User = CountUsers()
  163. stats.Counter.Org = CountOrganizations()
  164. stats.Counter.PublicKey, _ = x.Count(new(PublicKey))
  165. stats.Counter.Repo = CountRepositories(true)
  166. stats.Counter.Watch, _ = x.Count(new(Watch))
  167. stats.Counter.Star, _ = x.Count(new(Star))
  168. stats.Counter.Action, _ = x.Count(new(Action))
  169. stats.Counter.Access, _ = x.Count(new(Access))
  170. stats.Counter.Issue, _ = x.Count(new(Issue))
  171. stats.Counter.Comment, _ = x.Count(new(Comment))
  172. stats.Counter.Oauth = 0
  173. stats.Counter.Follow, _ = x.Count(new(Follow))
  174. stats.Counter.Mirror, _ = x.Count(new(Mirror))
  175. stats.Counter.Release, _ = x.Count(new(Release))
  176. stats.Counter.LoginSource = CountLoginSources()
  177. stats.Counter.Webhook, _ = x.Count(new(Webhook))
  178. stats.Counter.Milestone, _ = x.Count(new(Milestone))
  179. stats.Counter.Label, _ = x.Count(new(Label))
  180. stats.Counter.HookTask, _ = x.Count(new(HookTask))
  181. stats.Counter.Team, _ = x.Count(new(Team))
  182. stats.Counter.Attachment, _ = x.Count(new(Attachment))
  183. return
  184. }
  185. func Ping() error {
  186. return x.Ping()
  187. }
  188. // The version table. Should have only one row with id==1
  189. type Version struct {
  190. ID int64
  191. Version int64
  192. }
  193. // DumpDatabase dumps all data from database to file system in JSON format.
  194. func DumpDatabase(dirPath string) error {
  195. if err := os.MkdirAll(dirPath, os.ModePerm); err != nil {
  196. return err
  197. }
  198. // Purposely create a local variable to not modify global variable
  199. allTables := append(legacyTables, new(Version))
  200. allTables = append(allTables, tables...)
  201. for _, table := range allTables {
  202. tableName := strings.TrimPrefix(fmt.Sprintf("%T", table), "*db.")
  203. tableFile := path.Join(dirPath, tableName+".json")
  204. f, err := os.Create(tableFile)
  205. if err != nil {
  206. return fmt.Errorf("create JSON file: %v", err)
  207. }
  208. if err = x.Asc("id").Iterate(table, func(idx int, bean interface{}) (err error) {
  209. return jsoniter.NewEncoder(f).Encode(bean)
  210. }); err != nil {
  211. _ = f.Close()
  212. return fmt.Errorf("dump table '%s': %v", tableName, err)
  213. }
  214. _ = f.Close()
  215. }
  216. return nil
  217. }
  218. // ImportDatabase imports data from backup archive.
  219. func ImportDatabase(dirPath string, verbose bool) (err error) {
  220. snakeMapper := core.SnakeMapper{}
  221. skipInsertProcessors := map[string]bool{
  222. "mirror": true,
  223. "milestone": true,
  224. }
  225. // Purposely create a local variable to not modify global variable
  226. allTables := append(legacyTables, new(Version))
  227. allTables = append(allTables, tables...)
  228. for _, table := range allTables {
  229. tableName := strings.TrimPrefix(fmt.Sprintf("%T", table), "*db.")
  230. tableFile := path.Join(dirPath, tableName+".json")
  231. if !com.IsExist(tableFile) {
  232. continue
  233. }
  234. if verbose {
  235. log.Trace("Importing table '%s'...", tableName)
  236. }
  237. if err = x.DropTables(table); err != nil {
  238. return fmt.Errorf("drop table '%s': %v", tableName, err)
  239. } else if err = x.Sync2(table); err != nil {
  240. return fmt.Errorf("sync table '%s': %v", tableName, err)
  241. }
  242. f, err := os.Open(tableFile)
  243. if err != nil {
  244. return fmt.Errorf("open JSON file: %v", err)
  245. }
  246. rawTableName := x.TableName(table)
  247. _, isInsertProcessor := table.(xorm.BeforeInsertProcessor)
  248. scanner := bufio.NewScanner(f)
  249. for scanner.Scan() {
  250. switch bean := table.(type) {
  251. case *LoginSource:
  252. meta := make(map[string]interface{})
  253. if err = jsoniter.Unmarshal(scanner.Bytes(), &meta); err != nil {
  254. return fmt.Errorf("unmarshal to map: %v", err)
  255. }
  256. tp := LoginType(com.StrTo(com.ToStr(meta["Type"])).MustInt64())
  257. switch tp {
  258. case LoginLDAP, LoginDLDAP:
  259. bean.Cfg = new(LDAPConfig)
  260. case LoginSMTP:
  261. bean.Cfg = new(SMTPConfig)
  262. case LoginPAM:
  263. bean.Cfg = new(PAMConfig)
  264. case LoginGitHub:
  265. bean.Cfg = new(GitHubConfig)
  266. default:
  267. return fmt.Errorf("unrecognized login source type:: %v", tp)
  268. }
  269. table = bean
  270. }
  271. if err = jsoniter.Unmarshal(scanner.Bytes(), table); err != nil {
  272. return fmt.Errorf("unmarshal to struct: %v", err)
  273. }
  274. if _, err = x.Insert(table); err != nil {
  275. return fmt.Errorf("insert strcut: %v", err)
  276. }
  277. meta := make(map[string]interface{})
  278. if err = jsoniter.Unmarshal(scanner.Bytes(), &meta); err != nil {
  279. log.Error("Failed to unmarshal to map: %v", err)
  280. }
  281. // Reset created_unix back to the date save in archive because Insert method updates its value
  282. if isInsertProcessor && !skipInsertProcessors[rawTableName] {
  283. if _, err = x.Exec("UPDATE "+rawTableName+" SET created_unix=? WHERE id=?", meta["CreatedUnix"], meta["ID"]); err != nil {
  284. log.Error("Failed to reset 'created_unix': %v", err)
  285. }
  286. }
  287. switch rawTableName {
  288. case "milestone":
  289. if _, err = x.Exec("UPDATE "+rawTableName+" SET deadline_unix=?, closed_date_unix=? WHERE id=?", meta["DeadlineUnix"], meta["ClosedDateUnix"], meta["ID"]); err != nil {
  290. log.Error("Failed to reset 'milestone.deadline_unix', 'milestone.closed_date_unix': %v", err)
  291. }
  292. }
  293. }
  294. // PostgreSQL needs manually reset table sequence for auto increment keys
  295. if conf.UsePostgreSQL {
  296. rawTableName := snakeMapper.Obj2Table(tableName)
  297. seqName := rawTableName + "_id_seq"
  298. if _, err = x.Exec(fmt.Sprintf(`SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM "%s"), 1), false);`, seqName, rawTableName)); err != nil {
  299. return fmt.Errorf("reset table '%s' sequence: %v", rawTableName, err)
  300. }
  301. }
  302. }
  303. return nil
  304. }