http.go 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201
  1. // Copyright 2013 com authors
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License"): you may
  4. // not use this file except in compliance with the License. You may obtain
  5. // a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11. // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12. // License for the specific language governing permissions and limitations
  13. // under the License.
  14. package com
  15. import (
  16. "bytes"
  17. "encoding/json"
  18. "fmt"
  19. "io"
  20. "io/ioutil"
  21. "net/http"
  22. "os"
  23. "path"
  24. )
  25. type NotFoundError struct {
  26. Message string
  27. }
  28. func (e NotFoundError) Error() string {
  29. return e.Message
  30. }
  31. type RemoteError struct {
  32. Host string
  33. Err error
  34. }
  35. func (e *RemoteError) Error() string {
  36. return e.Err.Error()
  37. }
  38. var UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1541.0 Safari/537.36"
  39. // HttpCall makes HTTP method call.
  40. func HttpCall(client *http.Client, method, url string, header http.Header, body io.Reader) (io.ReadCloser, error) {
  41. req, err := http.NewRequest(method, url, body)
  42. if err != nil {
  43. return nil, err
  44. }
  45. req.Header.Set("User-Agent", UserAgent)
  46. for k, vs := range header {
  47. req.Header[k] = vs
  48. }
  49. resp, err := client.Do(req)
  50. if err != nil {
  51. return nil, err
  52. }
  53. if resp.StatusCode == 200 {
  54. return resp.Body, nil
  55. }
  56. resp.Body.Close()
  57. if resp.StatusCode == 404 { // 403 can be rate limit error. || resp.StatusCode == 403 {
  58. err = fmt.Errorf("resource not found: %s", url)
  59. } else {
  60. err = fmt.Errorf("%s %s -> %d", method, url, resp.StatusCode)
  61. }
  62. return nil, err
  63. }
  64. // HttpGet gets the specified resource.
  65. // ErrNotFound is returned if the server responds with status 404.
  66. func HttpGet(client *http.Client, url string, header http.Header) (io.ReadCloser, error) {
  67. return HttpCall(client, "GET", url, header, nil)
  68. }
  69. // HttpPost posts the specified resource.
  70. // ErrNotFound is returned if the server responds with status 404.
  71. func HttpPost(client *http.Client, url string, header http.Header, body []byte) (io.ReadCloser, error) {
  72. return HttpCall(client, "POST", url, header, bytes.NewBuffer(body))
  73. }
  74. // HttpGetToFile gets the specified resource and writes to file.
  75. // ErrNotFound is returned if the server responds with status 404.
  76. func HttpGetToFile(client *http.Client, url string, header http.Header, fileName string) error {
  77. rc, err := HttpGet(client, url, header)
  78. if err != nil {
  79. return err
  80. }
  81. defer rc.Close()
  82. os.MkdirAll(path.Dir(fileName), os.ModePerm)
  83. f, err := os.Create(fileName)
  84. if err != nil {
  85. return err
  86. }
  87. defer f.Close()
  88. _, err = io.Copy(f, rc)
  89. return err
  90. }
  91. // HttpGetBytes gets the specified resource. ErrNotFound is returned if the server
  92. // responds with status 404.
  93. func HttpGetBytes(client *http.Client, url string, header http.Header) ([]byte, error) {
  94. rc, err := HttpGet(client, url, header)
  95. if err != nil {
  96. return nil, err
  97. }
  98. defer rc.Close()
  99. return ioutil.ReadAll(rc)
  100. }
  101. // HttpGetJSON gets the specified resource and mapping to struct.
  102. // ErrNotFound is returned if the server responds with status 404.
  103. func HttpGetJSON(client *http.Client, url string, v interface{}) error {
  104. rc, err := HttpGet(client, url, nil)
  105. if err != nil {
  106. return err
  107. }
  108. defer rc.Close()
  109. err = json.NewDecoder(rc).Decode(v)
  110. if _, ok := err.(*json.SyntaxError); ok {
  111. return fmt.Errorf("JSON syntax error at %s", url)
  112. }
  113. return nil
  114. }
  115. // HttpPostJSON posts the specified resource with struct values,
  116. // and maps results to struct.
  117. // ErrNotFound is returned if the server responds with status 404.
  118. func HttpPostJSON(client *http.Client, url string, body, v interface{}) error {
  119. data, err := json.Marshal(body)
  120. if err != nil {
  121. return err
  122. }
  123. rc, err := HttpPost(client, url, http.Header{"content-type": []string{"application/json"}}, data)
  124. if err != nil {
  125. return err
  126. }
  127. defer rc.Close()
  128. err = json.NewDecoder(rc).Decode(v)
  129. if _, ok := err.(*json.SyntaxError); ok {
  130. return fmt.Errorf("JSON syntax error at %s", url)
  131. }
  132. return nil
  133. }
  134. // A RawFile describes a file that can be downloaded.
  135. type RawFile interface {
  136. Name() string
  137. RawUrl() string
  138. Data() []byte
  139. SetData([]byte)
  140. }
  141. // FetchFiles fetches files specified by the rawURL field in parallel.
  142. func FetchFiles(client *http.Client, files []RawFile, header http.Header) error {
  143. ch := make(chan error, len(files))
  144. for i := range files {
  145. go func(i int) {
  146. p, err := HttpGetBytes(client, files[i].RawUrl(), nil)
  147. if err != nil {
  148. ch <- err
  149. return
  150. }
  151. files[i].SetData(p)
  152. ch <- nil
  153. }(i)
  154. }
  155. for _ = range files {
  156. if err := <-ch; err != nil {
  157. return err
  158. }
  159. }
  160. return nil
  161. }
  162. // FetchFiles uses command `curl` to fetch files specified by the rawURL field in parallel.
  163. func FetchFilesCurl(files []RawFile, curlOptions ...string) error {
  164. ch := make(chan error, len(files))
  165. for i := range files {
  166. go func(i int) {
  167. stdout, _, err := ExecCmd("curl", append(curlOptions, files[i].RawUrl())...)
  168. if err != nil {
  169. ch <- err
  170. return
  171. }
  172. files[i].SetData([]byte(stdout))
  173. ch <- nil
  174. }(i)
  175. }
  176. for _ = range files {
  177. if err := <-ch; err != nil {
  178. return err
  179. }
  180. }
  181. return nil
  182. }