Go实现AWS4请求认证

　　什么是 Amazon S3？它是 AWS 提供的一种对象存储服务，提供行业领先的可扩展性、数据可用性、安全性和性能。Amazon S3 可达到 99.999999999%（11 个 9）的持久性。
　　Amazon S3 里用到了  AWS Signature Version 4  （下面简称 AWS4  ）做认证请求，这篇文章将会讲解如何使用 Go 实现 AWS4 请求认证。
　　转自：https://juejin.cn/post/6950300506946273294
　　整理：地鼠文档www.topgoer.cn
　　AWS4 是一种用于对所有 AWS 区域服务的入站 API 请求进行身份验证的协议。 AWS4
　　AWS4 对请求进行签名有以下优势（但这也取决于你如何使用）： 验证请求者的身份  - 经过身份验证的请求需要使用  AccessKeyID   和 SecretAccessKey   创建签名。保护传输中的数据  - 为了防止在传输过程中对请求进行篡改，可以使用一些请求元素（比如 请求路径  、请求头  等）来计算请求签名。Amazon S3 在收到请求后，使用相同的请求元素来计算签名。如果 Amazon S3 接收到的任何请求组件与用于计算签名的组件不匹配，Amazon S3 将拒绝该请求。防止重用请求的签名部分  - 请求的签名部分在请求中的时间戳的一段时间内有效。 授权方式HTTP 身份验证头，例如 Authorization 请求头： Authorization: AWS4-HMAC-SHA256  Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,  SignedHeaders=host;range;x-amz-date, Signature=fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024URL 查询字符串参数，例如预签名 URL： https://s3.amazonaws.com/examplebucket/test.txt ?X-Amz-Algorithm=AWS4-HMAC-SHA256 &X-Amz-Credential=/20130721/us-east-1/s3/aws4_request &X-Amz-Date=20130721T201207Z &X-Amz-Expires=86400 &X-Amz-SignedHeaders=host &X-Amz-Signature=Go 实现 HTTP 身份验证头HTTP 身份验证头组成部分AWS4-HMAC-SHA256  - 该字符串指定 AWS4 和签名算法（HMAC-SHA256）。 Credential  - 指定 AccessKeyID、计算签名的日期、区域和服务。它的格式是 ////aws4_request  ，date   的格式是 YYYYMMDD  。SignedHeaders  - 指定用于计算签名的请求头列表，以分号分隔。仅包含请求头的名称，且必须是小写，例如： host;range;x-amz-date  。Signature  - 表示为 64 个小写十六进制字符的 256 位签名，例如： fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024  。上传图片至掘金
　　根据对 AWS4 请求认证的学习，并参考 simples3 的代码实现了 上传图片至掘金（字节跳动的存储服务，其服务名称是 imagex  ）的功能，下面是部分的代码实现（主要省略了Client   部分的实现，完整代码有待后续完善后进行开源）：package juejin  import (     ＂bytes＂     ＂crypto/hmac＂     ＂crypto/sha256＂     ＂encoding/hex＂     ＂encoding/json＂     ＂fmt＂     ＂hash/crc32＂     ＂io＂     ＂io/ioutil＂     ＂net/http＂     ＂net/url＂     ＂os＂     ＂path/filepath＂     ＂regexp＂     ＂sort＂     ＂strings＂     ＂time＂      ＂github.com/tidwall/gjson＂ )  const (     amzDateISO8601TimeFormat = ＂20060102T150405Z＂     shortTimeFormat          = ＂20060102＂     algorithm                = ＂AWS4-HMAC-SHA256＂     serviceName              = ＂imagex＂     serviceID                = ＂k3u1fbpfcp＂     version                  = ＂2018-08-01＂     uploadURLFormat          = ＂https://%s/%s＂      RegionCNNorth = ＂cn-north-1＂      actionApplyImageUpload  = ＂ApplyImageUpload＂     actionCommitImageUpload = ＂CommitImageUpload＂      polynomialCRC32 = 0xEDB88320 )  var (     newLine = []byte{＂ ＂}      // if object matches reserved string, no need to encode them     reservedObjectNames = regexp.MustCompile(＂^[a-zA-Z0-9-_.~/]+#34;) )  type ImageX struct {     AccessKey string     SecretKey string     Region    string     Client    *http.Client      Token   string     Version string     BaseURL string }  type UploadToken struct {     AccessKeyID     string `json:＂AccessKeyID＂`     SecretAccessKey string `json:＂SecretAccessKey＂`     SessionToken    string `json:＂SessionToken＂` }  func (c *Client) UploadImage(region, imgPath string) (string, error) {     uploadToken, err := c.GetUploadToken()     if err != nil {         return ＂＂, err     }      ix := &ImageX{         AccessKey: uploadToken.AccessKeyID,         SecretKey: uploadToken.SecretAccessKey,         Token:     uploadToken.SessionToken,         Region:    region,     }      applyRes, err := ix.ApplyImageUpload()     if err != nil {         return ＂＂, err     }      storeInfo := gjson.Get(applyRes, ＂Result.UploadAddress.StoreInfos.0＂)     storeURI := storeInfo.Get(＂StoreUri＂).String()     storeAuth := storeInfo.Get(＂Auth＂).String()     uploadHost := gjson.Get(applyRes, ＂Result.UploadAddress.UploadHosts.0＂).String()     uploadURL := fmt.Sprintf(uploadURLFormat, uploadHost, storeURI)     if err := ix.Upload(uploadURL, imgPath, storeAuth); err != nil {         return ＂＂, err     }      sessionKey := gjson.Get(applyRes, ＂Result.UploadAddress.SessionKey＂).String()     if _, err = ix.CommitImageUpload(sessionKey); err != nil {         return ＂＂, err     }      return c.GetImageURL(storeURI) }  func (c *Client) GetImageURL(uri string) (string, error) {     endpoint := ＂/imagex/get_img_url＂     params := &url.Values{         ＂uri＂: []string{uri},     }     raw, err := c.Get(APIBaseURL, endpoint, params)     if err != nil {         return ＂＂, err     }     rawurl := gjson.Get(raw, ＂data.main_url＂).String()     return rawurl, nil }  func (c *Client) GetUploadToken() (*UploadToken, error) {     endpoint := ＂/imagex/gen_token＂     params := &url.Values{         ＂client＂: []string{＂web＂},     }     raw, err := c.Get(APIBaseURL, endpoint, params)     if err != nil {         return nil, err     }     var token *UploadToken     err = json.Unmarshal([]byte(gjson.Get(raw, ＂data.token＂).String()), &token)     return token, err }  func (ix *ImageX) ApplyImageUpload() (string, error) {     rawurl := fmt.Sprintf(＂https://imagex.bytedanceapi.com/?Action=%s&Version=%s&ServiceId=%s＂,         actionApplyImageUpload, version, serviceID)     req, err := http.NewRequest(http.MethodGet, rawurl, nil)     if err != nil {         return ＂＂, err     }      if err := ix.signRequest(req); err != nil {         return ＂＂, err     }      res, err := ix.getClient().Do(req)     if err != nil {         return ＂＂, err     }     defer res.Body.Close()     b, err := ioutil.ReadAll(res.Body)     if err != nil {         return ＂＂, err     }     raw := string(b)     if res.StatusCode != 200 || gjson.Get(raw, ＂ResponseMetadata.Error＂).Exists() {         return ＂＂, fmt.Errorf(＂raw: %s, response: %+v＂, raw, res)     }     return raw, nil }  func (ix *ImageX) CommitImageUpload(sessionKey string) (string, error) {     rawurl := fmt.Sprintf(＂https://imagex.bytedanceapi.com/?Action=%s&Version=%s&SessionKey=%s&ServiceId=%s＂,         actionCommitImageUpload, version, sessionKey, serviceID)     req, err := http.NewRequest(http.MethodPost, rawurl, nil)     if err != nil {         return ＂＂, err     }      if err := ix.signRequest(req); err != nil {         return ＂＂, err     }      res, err := ix.getClient().Do(req)     if err != nil {         return ＂＂, err     }     defer res.Body.Close()     b, err := ioutil.ReadAll(res.Body)     if err != nil {         return ＂＂, err     }     raw := string(b)     if res.StatusCode != 200 || gjson.Get(raw, ＂ResponseMetadata.Error＂).Exists() {         return ＂＂, fmt.Errorf(＂raw: %s, response: %+v＂, raw, res)     }     return raw, nil }  func (ix *ImageX) getClient() *http.Client {     if ix.Client == nil {         return http.DefaultClient     }     return ix.Client }  func (ix *ImageX) signKeys(t time.Time) []byte {     h := makeHMac([]byte(＂AWS4＂+ix.SecretKey), []byte(t.Format(shortTimeFormat)))     h = makeHMac(h, []byte(ix.Region))     h = makeHMac(h, []byte(serviceName))     h = makeHMac(h, []byte(＂aws4_request＂))     return h }  func (ix *ImageX) writeRequest(w io.Writer, r *http.Request) error {     r.Header.Set(＂host＂, r.Host)      w.Write([]byte(r.Method))     w.Write(newLine)     writeURI(w, r)     w.Write(newLine)     writeQuery(w, r)     w.Write(newLine)     writeHeader(w, r)     w.Write(newLine)     w.Write(newLine)     writeHeaderList(w, r)     w.Write(newLine)     return writeBody(w, r) }  func (ix *ImageX) writeStringToSign(w io.Writer, t time.Time, r *http.Request) error {     w.Write([]byte(algorithm))     w.Write(newLine)     w.Write([]byte(t.Format(amzDateISO8601TimeFormat)))     w.Write(newLine)      w.Write([]byte(ix.creds(t)))     w.Write(newLine)      h := sha256.New()     if err := ix.writeRequest(h, r); err != nil {         return err     }     fmt.Fprintf(w, ＂%x＂, h.Sum(nil))     return nil }  func (ix *ImageX) creds(t time.Time) string {     return t.Format(shortTimeFormat) + ＂/＂ + ix.Region + ＂/＂ + serviceName + ＂/aws4_request＂ }  func (ix *ImageX) signRequest(req *http.Request) error {     t := time.Now().UTC()     req.Header.Set(＂x-amz-date＂, t.Format(amzDateISO8601TimeFormat))      req.Header.Set(＂x-amz-security-token＂, ix.Token)      k := ix.signKeys(t)     h := hmac.New(sha256.New, k)      if err := ix.writeStringToSign(h, t, req); err != nil {         return err     }      auth := bytes.NewBufferString(algorithm)     auth.Write([]byte(＂ Credential=＂ + ix.AccessKey + ＂/＂ + ix.creds(t)))     auth.Write([]byte{＂,＂, ＂ ＂})     auth.Write([]byte(＂SignedHeaders=＂))     writeHeaderList(auth, req)     auth.Write([]byte{＂,＂, ＂ ＂})     auth.Write([]byte(＂Signature=＂ + fmt.Sprintf(＂%x＂, h.Sum(nil))))      req.Header.Set(＂authorization＂, auth.String())     return nil }  func writeURI(w io.Writer, r *http.Request) {     path := r.URL.RequestURI()     if r.URL.RawQuery != ＂＂ {         path = path[:len(path)-len(r.URL.RawQuery)-1]     }     slash := strings.HasSuffix(path, ＂/＂)     path = filepath.Clean(path)     if path != ＂/＂ && slash {         path += ＂/＂     }     w.Write([]byte(path)) } func writeQuery(w io.Writer, r *http.Request) {     var a []string     for k, vs := range r.URL.Query() {         k = url.QueryEscape(k)         for _, v := range vs {             if v == ＂＂ {                 a = append(a, k)             } else {                 v = url.QueryEscape(v)                 a = append(a, k+＂=＂+v)             }         }     }     sort.Strings(a)     for i, s := range a {         if i > 0 {             w.Write([]byte{＂&＂})         }         w.Write([]byte(s))     } }  func writeHeader(w io.Writer, r *http.Request) {     i, a := 0, make([]string, len(r.Header))     for k, v := range r.Header {         sort.Strings(v)         a[i] = strings.ToLower(k) + ＂:＂ + strings.Join(v, ＂,＂)         i++     }     sort.Strings(a)     for i, s := range a {         if i > 0 {             w.Write(newLine)         }         io.WriteString(w, s)     } }  func writeHeaderList(w io.Writer, r *http.Request) {     i, a := 0, make([]string, len(r.Header))     for k := range r.Header {         a[i] = strings.ToLower(k)         i++     }     sort.Strings(a)     for i, s := range a {         if i > 0 {             w.Write([]byte{＂;＂})         }         w.Write([]byte(s))     } }  func writeBody(w io.Writer, r *http.Request) error {     var (         b   []byte         err error     )     // If the payload is empty, use the empty string as the input to the SHA256 function     // http://docs.amazonwebservices.com/general/latest/gr/sigv4-create-canonical-request.html     if r.Body == nil {         b = []byte(＂＂)     } else {         b, err = ioutil.ReadAll(r.Body)         if err != nil {             return err         }         r.Body = ioutil.NopCloser(bytes.NewBuffer(b))     }      h := sha256.New()     h.Write(b)     fmt.Fprintf(w, ＂%x＂, h.Sum(nil))     return nil }  func makeHMac(key []byte, data []byte) []byte {     hash := hmac.New(sha256.New, key)     hash.Write(data)     return hash.Sum(nil) }  func (ix *ImageX) Upload(rawurl, fp, auth string) error {     crc32, err := hashFileCRC32(fp)     if err != nil {         return err     }     file, err := os.Open(fp)     if err != nil {         return err     }     defer file.Close()      req, err := http.NewRequest(http.MethodPost, rawurl, file)     if err != nil {         return err     }     req.Header.Add(＂authorization＂, auth)     req.Header.Add(＂Content-Type＂, ＂application/octet-stream＂)     req.Header.Add(＂content-crc32＂, crc32)     res, err := http.DefaultClient.Do(req)     if err != nil {         return err     }     defer res.Body.Close()     b, err := ioutil.ReadAll(res.Body)     if err != nil {         return err     }     raw := string(b)     if gjson.Get(raw, ＂success＂).Int() != 0 {         return fmt.Errorf(＂raw: %s, response: %+v＂, raw, res)     }     return nil }  // hashFileCRC32 generate CRC32 hash of a file // Refer https://mrwaggel.be/post/generate-crc32-hash-of-a-file-in-golang-turorial/ func hashFileCRC32(filePath string) (string, error) {     file, err := os.Open(filePath)     if err != nil {         return ＂＂, err     }     defer file.Close()     tablePolynomial := crc32.MakeTable(polynomialCRC32)     hash := crc32.New(tablePolynomial)     if _, err := io.Copy(hash, file); err != nil {         return ＂＂, err     }     return hex.EncodeToString(hash.Sum(nil)), nil }