webhook.go 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663
  1. // Copyright 2014 The Gogs Authors. All rights reserved.
  2. // Use of this source code is governed by a MIT-style
  3. // license that can be found in the LICENSE file.
  4. package models
  5. import (
  6. "crypto/tls"
  7. "encoding/json"
  8. "fmt"
  9. "io/ioutil"
  10. "strings"
  11. "sync"
  12. "time"
  13. "github.com/Unknwon/com"
  14. "github.com/go-xorm/xorm"
  15. gouuid "github.com/satori/go.uuid"
  16. api "github.com/gogits/go-gogs-client"
  17. "github.com/gogits/gogs/modules/httplib"
  18. "github.com/gogits/gogs/modules/log"
  19. "github.com/gogits/gogs/modules/setting"
  20. )
  21. type HookContentType int
  22. const (
  23. JSON HookContentType = iota + 1
  24. FORM
  25. )
  26. var hookContentTypes = map[string]HookContentType{
  27. "json": JSON,
  28. "form": FORM,
  29. }
  30. // ToHookContentType returns HookContentType by given name.
  31. func ToHookContentType(name string) HookContentType {
  32. return hookContentTypes[name]
  33. }
  34. func (t HookContentType) Name() string {
  35. switch t {
  36. case JSON:
  37. return "json"
  38. case FORM:
  39. return "form"
  40. }
  41. return ""
  42. }
  43. // IsValidHookContentType returns true if given name is a valid hook content type.
  44. func IsValidHookContentType(name string) bool {
  45. _, ok := hookContentTypes[name]
  46. return ok
  47. }
  48. type HookEvents struct {
  49. Create bool `json:"create"`
  50. Push bool `json:"push"`
  51. }
  52. // HookEvent represents events that will delivery hook.
  53. type HookEvent struct {
  54. PushOnly bool `json:"push_only"`
  55. SendEverything bool `json:"send_everything"`
  56. ChooseEvents bool `json:"choose_events"`
  57. HookEvents `json:"events"`
  58. }
  59. type HookStatus int
  60. const (
  61. HOOK_STATUS_NONE = iota
  62. HOOK_STATUS_SUCCEED
  63. HOOK_STATUS_FAILED
  64. )
  65. // Webhook represents a web hook object.
  66. type Webhook struct {
  67. ID int64 `xorm:"pk autoincr"`
  68. RepoID int64
  69. OrgID int64
  70. URL string `xorm:"url TEXT"`
  71. ContentType HookContentType
  72. Secret string `xorm:"TEXT"`
  73. Events string `xorm:"TEXT"`
  74. *HookEvent `xorm:"-"`
  75. IsSSL bool `xorm:"is_ssl"`
  76. IsActive bool
  77. HookTaskType HookTaskType
  78. Meta string `xorm:"TEXT"` // store hook-specific attributes
  79. LastStatus HookStatus // Last delivery status
  80. Created time.Time `xorm:"-"`
  81. CreatedUnix int64
  82. Updated time.Time `xorm:"-"`
  83. UpdatedUnix int64
  84. }
  85. func (w *Webhook) BeforeInsert() {
  86. w.CreatedUnix = time.Now().Unix()
  87. w.UpdatedUnix = w.CreatedUnix
  88. }
  89. func (w *Webhook) BeforeUpdate() {
  90. w.UpdatedUnix = time.Now().Unix()
  91. }
  92. func (w *Webhook) AfterSet(colName string, _ xorm.Cell) {
  93. var err error
  94. switch colName {
  95. case "events":
  96. w.HookEvent = &HookEvent{}
  97. if err = json.Unmarshal([]byte(w.Events), w.HookEvent); err != nil {
  98. log.Error(3, "Unmarshal[%d]: %v", w.ID, err)
  99. }
  100. case "created_unix":
  101. w.Created = time.Unix(w.CreatedUnix, 0).Local()
  102. case "updated_unix":
  103. w.Updated = time.Unix(w.UpdatedUnix, 0).Local()
  104. }
  105. }
  106. func (w *Webhook) GetSlackHook() *SlackMeta {
  107. s := &SlackMeta{}
  108. if err := json.Unmarshal([]byte(w.Meta), s); err != nil {
  109. log.Error(4, "webhook.GetSlackHook(%d): %v", w.ID, err)
  110. }
  111. return s
  112. }
  113. // History returns history of webhook by given conditions.
  114. func (w *Webhook) History(page int) ([]*HookTask, error) {
  115. return HookTasks(w.ID, page)
  116. }
  117. // UpdateEvent handles conversion from HookEvent to Events.
  118. func (w *Webhook) UpdateEvent() error {
  119. data, err := json.Marshal(w.HookEvent)
  120. w.Events = string(data)
  121. return err
  122. }
  123. // HasCreateEvent returns true if hook enabled create event.
  124. func (w *Webhook) HasCreateEvent() bool {
  125. return w.SendEverything ||
  126. (w.ChooseEvents && w.HookEvents.Create)
  127. }
  128. // HasPushEvent returns true if hook enabled push event.
  129. func (w *Webhook) HasPushEvent() bool {
  130. return w.PushOnly || w.SendEverything ||
  131. (w.ChooseEvents && w.HookEvents.Push)
  132. }
  133. func (w *Webhook) EventsArray() []string {
  134. events := make([]string, 0, 2)
  135. if w.HasCreateEvent() {
  136. events = append(events, "create")
  137. }
  138. if w.HasPushEvent() {
  139. events = append(events, "push")
  140. }
  141. return events
  142. }
  143. // CreateWebhook creates a new web hook.
  144. func CreateWebhook(w *Webhook) error {
  145. _, err := x.Insert(w)
  146. return err
  147. }
  148. // getWebhook uses argument bean as query condition,
  149. // ID must be specified and do not assign unnecessary fields.
  150. func getWebhook(bean *Webhook) (*Webhook, error) {
  151. has, err := x.Get(bean)
  152. if err != nil {
  153. return nil, err
  154. } else if !has {
  155. return nil, ErrWebhookNotExist{bean.ID}
  156. }
  157. return bean, nil
  158. }
  159. // GetWebhookByRepoID returns webhook of repository by given ID.
  160. func GetWebhookByRepoID(repoID, id int64) (*Webhook, error) {
  161. return getWebhook(&Webhook{
  162. ID: id,
  163. RepoID: repoID,
  164. })
  165. }
  166. // GetWebhookByOrgID returns webhook of organization by given ID.
  167. func GetWebhookByOrgID(orgID, id int64) (*Webhook, error) {
  168. return getWebhook(&Webhook{
  169. ID: id,
  170. OrgID: orgID,
  171. })
  172. }
  173. // GetActiveWebhooksByRepoID returns all active webhooks of repository.
  174. func GetActiveWebhooksByRepoID(repoID int64) (ws []*Webhook, err error) {
  175. err = x.Where("repo_id=?", repoID).And("is_active=?", true).Find(&ws)
  176. return ws, err
  177. }
  178. // GetWebhooksByRepoID returns all webhooks of repository.
  179. func GetWebhooksByRepoID(repoID int64) (ws []*Webhook, err error) {
  180. err = x.Find(&ws, &Webhook{RepoID: repoID})
  181. return ws, err
  182. }
  183. // UpdateWebhook updates information of webhook.
  184. func UpdateWebhook(w *Webhook) error {
  185. _, err := x.Id(w.ID).AllCols().Update(w)
  186. return err
  187. }
  188. // deleteWebhook uses argument bean as query condition,
  189. // ID must be specified and do not assign unnecessary fields.
  190. func deleteWebhook(bean *Webhook) (err error) {
  191. sess := x.NewSession()
  192. defer sessionRelease(sess)
  193. if err = sess.Begin(); err != nil {
  194. return err
  195. }
  196. if _, err = sess.Delete(bean); err != nil {
  197. return err
  198. } else if _, err = sess.Delete(&HookTask{HookID: bean.ID}); err != nil {
  199. return err
  200. }
  201. return sess.Commit()
  202. }
  203. // DeleteWebhookByRepoID deletes webhook of repository by given ID.
  204. func DeleteWebhookByRepoID(repoID, id int64) error {
  205. return deleteWebhook(&Webhook{
  206. ID: id,
  207. RepoID: repoID,
  208. })
  209. }
  210. // DeleteWebhookByOrgID deletes webhook of organization by given ID.
  211. func DeleteWebhookByOrgID(orgID, id int64) error {
  212. return deleteWebhook(&Webhook{
  213. ID: id,
  214. OrgID: orgID,
  215. })
  216. }
  217. // GetWebhooksByOrgID returns all webhooks for an organization.
  218. func GetWebhooksByOrgID(orgID int64) (ws []*Webhook, err error) {
  219. err = x.Find(&ws, &Webhook{OrgID: orgID})
  220. return ws, err
  221. }
  222. // GetActiveWebhooksByOrgID returns all active webhooks for an organization.
  223. func GetActiveWebhooksByOrgID(orgID int64) (ws []*Webhook, err error) {
  224. err = x.Where("org_id=?", orgID).And("is_active=?", true).Find(&ws)
  225. return ws, err
  226. }
  227. // ___ ___ __ ___________ __
  228. // / | \ ____ ____ | | _\__ ___/____ _____| | __
  229. // / ~ \/ _ \ / _ \| |/ / | | \__ \ / ___/ |/ /
  230. // \ Y ( <_> | <_> ) < | | / __ \_\___ \| <
  231. // \___|_ / \____/ \____/|__|_ \ |____| (____ /____ >__|_ \
  232. // \/ \/ \/ \/ \/
  233. type HookTaskType int
  234. const (
  235. GOGS HookTaskType = iota + 1
  236. SLACK
  237. )
  238. var hookTaskTypes = map[string]HookTaskType{
  239. "gogs": GOGS,
  240. "slack": SLACK,
  241. }
  242. // ToHookTaskType returns HookTaskType by given name.
  243. func ToHookTaskType(name string) HookTaskType {
  244. return hookTaskTypes[name]
  245. }
  246. func (t HookTaskType) Name() string {
  247. switch t {
  248. case GOGS:
  249. return "gogs"
  250. case SLACK:
  251. return "slack"
  252. }
  253. return ""
  254. }
  255. // IsValidHookTaskType returns true if given name is a valid hook task type.
  256. func IsValidHookTaskType(name string) bool {
  257. _, ok := hookTaskTypes[name]
  258. return ok
  259. }
  260. type HookEventType string
  261. const (
  262. HOOK_EVENT_CREATE HookEventType = "create"
  263. HOOK_EVENT_PUSH HookEventType = "push"
  264. )
  265. // HookRequest represents hook task request information.
  266. type HookRequest struct {
  267. Headers map[string]string `json:"headers"`
  268. }
  269. // HookResponse represents hook task response information.
  270. type HookResponse struct {
  271. Status int `json:"status"`
  272. Headers map[string]string `json:"headers"`
  273. Body string `json:"body"`
  274. }
  275. // HookTask represents a hook task.
  276. type HookTask struct {
  277. ID int64 `xorm:"pk autoincr"`
  278. RepoID int64 `xorm:"INDEX"`
  279. HookID int64
  280. UUID string
  281. Type HookTaskType
  282. URL string `xorm:"TEXT"`
  283. api.Payloader `xorm:"-"`
  284. PayloadContent string `xorm:"TEXT"`
  285. ContentType HookContentType
  286. EventType HookEventType
  287. IsSSL bool
  288. IsDelivered bool
  289. Delivered int64
  290. DeliveredString string `xorm:"-"`
  291. // History info.
  292. IsSucceed bool
  293. RequestContent string `xorm:"TEXT"`
  294. RequestInfo *HookRequest `xorm:"-"`
  295. ResponseContent string `xorm:"TEXT"`
  296. ResponseInfo *HookResponse `xorm:"-"`
  297. }
  298. func (t *HookTask) BeforeUpdate() {
  299. if t.RequestInfo != nil {
  300. t.RequestContent = t.MarshalJSON(t.RequestInfo)
  301. }
  302. if t.ResponseInfo != nil {
  303. t.ResponseContent = t.MarshalJSON(t.ResponseInfo)
  304. }
  305. }
  306. func (t *HookTask) AfterSet(colName string, _ xorm.Cell) {
  307. var err error
  308. switch colName {
  309. case "delivered":
  310. t.DeliveredString = time.Unix(0, t.Delivered).Format("2006-01-02 15:04:05 MST")
  311. case "request_content":
  312. if len(t.RequestContent) == 0 {
  313. return
  314. }
  315. t.RequestInfo = &HookRequest{}
  316. if err = json.Unmarshal([]byte(t.RequestContent), t.RequestInfo); err != nil {
  317. log.Error(3, "Unmarshal[%d]: %v", t.ID, err)
  318. }
  319. case "response_content":
  320. if len(t.ResponseContent) == 0 {
  321. return
  322. }
  323. t.ResponseInfo = &HookResponse{}
  324. if err = json.Unmarshal([]byte(t.ResponseContent), t.ResponseInfo); err != nil {
  325. log.Error(3, "Unmarshal [%d]: %v", t.ID, err)
  326. }
  327. }
  328. }
  329. func (t *HookTask) MarshalJSON(v interface{}) string {
  330. p, err := json.Marshal(v)
  331. if err != nil {
  332. log.Error(3, "Marshal [%d]: %v", t.ID, err)
  333. }
  334. return string(p)
  335. }
  336. // HookTasks returns a list of hook tasks by given conditions.
  337. func HookTasks(hookID int64, page int) ([]*HookTask, error) {
  338. tasks := make([]*HookTask, 0, setting.Webhook.PagingNum)
  339. return tasks, x.Limit(setting.Webhook.PagingNum, (page-1)*setting.Webhook.PagingNum).Where("hook_id=?", hookID).Desc("id").Find(&tasks)
  340. }
  341. // CreateHookTask creates a new hook task,
  342. // it handles conversion from Payload to PayloadContent.
  343. func CreateHookTask(t *HookTask) error {
  344. data, err := t.Payloader.JSONPayload()
  345. if err != nil {
  346. return err
  347. }
  348. t.UUID = gouuid.NewV4().String()
  349. t.PayloadContent = string(data)
  350. _, err = x.Insert(t)
  351. return err
  352. }
  353. // UpdateHookTask updates information of hook task.
  354. func UpdateHookTask(t *HookTask) error {
  355. _, err := x.Id(t.ID).AllCols().Update(t)
  356. return err
  357. }
  358. // PrepareWebhooks adds new webhooks to task queue for given payload.
  359. func PrepareWebhooks(repo *Repository, event HookEventType, p api.Payloader) error {
  360. if err := repo.GetOwner(); err != nil {
  361. return fmt.Errorf("GetOwner: %v", err)
  362. }
  363. ws, err := GetActiveWebhooksByRepoID(repo.ID)
  364. if err != nil {
  365. return fmt.Errorf("GetActiveWebhooksByRepoID: %v", err)
  366. }
  367. // check if repo belongs to org and append additional webhooks
  368. if repo.Owner.IsOrganization() {
  369. // get hooks for org
  370. orgws, err := GetActiveWebhooksByOrgID(repo.OwnerID)
  371. if err != nil {
  372. return fmt.Errorf("GetActiveWebhooksByOrgID: %v", err)
  373. }
  374. ws = append(ws, orgws...)
  375. }
  376. if len(ws) == 0 {
  377. return nil
  378. }
  379. var payloader api.Payloader
  380. for _, w := range ws {
  381. switch event {
  382. case HOOK_EVENT_CREATE:
  383. if !w.HasCreateEvent() {
  384. continue
  385. }
  386. case HOOK_EVENT_PUSH:
  387. if !w.HasPushEvent() {
  388. continue
  389. }
  390. }
  391. // Use separate objects so modifcations won't be made on payload on non-Gogs type hooks.
  392. switch w.HookTaskType {
  393. case SLACK:
  394. payloader, err = GetSlackPayload(p, event, w.Meta)
  395. if err != nil {
  396. return fmt.Errorf("GetSlackPayload: %v", err)
  397. }
  398. default:
  399. p.SetSecret(w.Secret)
  400. payloader = p
  401. }
  402. if err = CreateHookTask(&HookTask{
  403. RepoID: repo.ID,
  404. HookID: w.ID,
  405. Type: w.HookTaskType,
  406. URL: w.URL,
  407. Payloader: payloader,
  408. ContentType: w.ContentType,
  409. EventType: HOOK_EVENT_PUSH,
  410. IsSSL: w.IsSSL,
  411. }); err != nil {
  412. return fmt.Errorf("CreateHookTask: %v", err)
  413. }
  414. }
  415. return nil
  416. }
  417. // UniqueQueue represents a queue that guarantees only one instance of same ID is in the line.
  418. type UniqueQueue struct {
  419. lock sync.Mutex
  420. ids map[string]bool
  421. queue chan string
  422. }
  423. func (q *UniqueQueue) Queue() <-chan string {
  424. return q.queue
  425. }
  426. func NewUniqueQueue(queueLength int) *UniqueQueue {
  427. if queueLength <= 0 {
  428. queueLength = 100
  429. }
  430. return &UniqueQueue{
  431. ids: make(map[string]bool),
  432. queue: make(chan string, queueLength),
  433. }
  434. }
  435. func (q *UniqueQueue) Remove(id interface{}) {
  436. q.lock.Lock()
  437. defer q.lock.Unlock()
  438. delete(q.ids, com.ToStr(id))
  439. }
  440. func (q *UniqueQueue) AddFunc(id interface{}, fn func()) {
  441. newid := com.ToStr(id)
  442. if q.Exist(id) {
  443. return
  444. }
  445. q.lock.Lock()
  446. q.ids[newid] = true
  447. if fn != nil {
  448. fn()
  449. }
  450. q.lock.Unlock()
  451. q.queue <- newid
  452. }
  453. func (q *UniqueQueue) Add(id interface{}) {
  454. q.AddFunc(id, nil)
  455. }
  456. func (q *UniqueQueue) Exist(id interface{}) bool {
  457. q.lock.Lock()
  458. defer q.lock.Unlock()
  459. return q.ids[com.ToStr(id)]
  460. }
  461. var HookQueue = NewUniqueQueue(setting.Webhook.QueueLength)
  462. func (t *HookTask) deliver() {
  463. t.IsDelivered = true
  464. timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second
  465. req := httplib.Post(t.URL).SetTimeout(timeout, timeout).
  466. Header("X-Gogs-Delivery", t.UUID).
  467. Header("X-Gogs-Event", string(t.EventType)).
  468. SetTLSClientConfig(&tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify})
  469. switch t.ContentType {
  470. case JSON:
  471. req = req.Header("Content-Type", "application/json").Body(t.PayloadContent)
  472. case FORM:
  473. req.Param("payload", t.PayloadContent)
  474. }
  475. // Record delivery information.
  476. t.RequestInfo = &HookRequest{
  477. Headers: map[string]string{},
  478. }
  479. for k, vals := range req.Headers() {
  480. t.RequestInfo.Headers[k] = strings.Join(vals, ",")
  481. }
  482. t.ResponseInfo = &HookResponse{
  483. Headers: map[string]string{},
  484. }
  485. defer func() {
  486. t.Delivered = time.Now().UnixNano()
  487. if t.IsSucceed {
  488. log.Trace("Hook delivered: %s", t.UUID)
  489. } else {
  490. log.Trace("Hook delivery failed: %s", t.UUID)
  491. }
  492. // Update webhook last delivery status.
  493. w, err := GetWebhookByRepoID(t.RepoID, t.HookID)
  494. if err != nil {
  495. log.Error(5, "GetWebhookByID: %v", err)
  496. return
  497. }
  498. if t.IsSucceed {
  499. w.LastStatus = HOOK_STATUS_SUCCEED
  500. } else {
  501. w.LastStatus = HOOK_STATUS_FAILED
  502. }
  503. if err = UpdateWebhook(w); err != nil {
  504. log.Error(5, "UpdateWebhook: %v", err)
  505. return
  506. }
  507. }()
  508. resp, err := req.Response()
  509. if err != nil {
  510. t.ResponseInfo.Body = fmt.Sprintf("Delivery: %v", err)
  511. return
  512. }
  513. defer resp.Body.Close()
  514. // Status code is 20x can be seen as succeed.
  515. t.IsSucceed = resp.StatusCode/100 == 2
  516. t.ResponseInfo.Status = resp.StatusCode
  517. for k, vals := range resp.Header {
  518. t.ResponseInfo.Headers[k] = strings.Join(vals, ",")
  519. }
  520. p, err := ioutil.ReadAll(resp.Body)
  521. if err != nil {
  522. t.ResponseInfo.Body = fmt.Sprintf("read body: %s", err)
  523. return
  524. }
  525. t.ResponseInfo.Body = string(p)
  526. }
  527. // DeliverHooks checks and delivers undelivered hooks.
  528. // TODO: shoot more hooks at same time.
  529. func DeliverHooks() {
  530. tasks := make([]*HookTask, 0, 10)
  531. x.Where("is_delivered=?", false).Iterate(new(HookTask),
  532. func(idx int, bean interface{}) error {
  533. t := bean.(*HookTask)
  534. t.deliver()
  535. tasks = append(tasks, t)
  536. return nil
  537. })
  538. // Update hook task status.
  539. for _, t := range tasks {
  540. if err := UpdateHookTask(t); err != nil {
  541. log.Error(4, "UpdateHookTask [%d]: %v", t.ID, err)
  542. }
  543. }
  544. // Start listening on new hook requests.
  545. for repoID := range HookQueue.Queue() {
  546. log.Trace("DeliverHooks [%v]: processing delivery hooks", repoID)
  547. HookQueue.Remove(repoID)
  548. tasks = make([]*HookTask, 0, 5)
  549. if err := x.Where("repo_id=? AND is_delivered=?", repoID, false).Find(&tasks); err != nil {
  550. log.Error(4, "Get repository [%d] hook tasks: %v", repoID, err)
  551. continue
  552. }
  553. for _, t := range tasks {
  554. t.deliver()
  555. if err := UpdateHookTask(t); err != nil {
  556. log.Error(4, "UpdateHookTask [%d]: %v", t.ID, err)
  557. continue
  558. }
  559. }
  560. }
  561. }
  562. func InitDeliverHooks() {
  563. go DeliverHooks()
  564. }