使用bash从S3下载私有文件

Question

我试图让以下bash脚本工作(从http://curl.haxx.se/mail/archive-2014-10/0006.html#replies复制):

#!/bin/sh 
file=path/to/file 
bucket=your-bucket 
resource="/${bucket}/${file}" 
contentType="application/x-compressed-tar" 
dateValue="`date +'%a, %d %b %Y %H:%M:%S %z'`" 
stringToSign="GET 
${contentType} 
${dateValue} 
${resource}" 
s3Key=xxxxxxxxxxxxxxxxxxxx 
s3Secret=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 
signature=`/bin/echo -n "$stringToSign" | openssl sha1 -hmac ${s3Secret} -binary |      base64` 
curl -H "Host: ${bucket}.s3.amazonaws.com" \
-H "Date: ${dateValue}" \
-H "Content-Type: ${contentType}" \ 
-H "Authorization: AWS ${s3Key}:${signature}" \ 
https://${bucket}.s3.amazonaws.com/${file}

无论我做什么,我都会收到SignatureDoesNotMatch错误.

任何关于如何解决这个问题的想法将不胜感激.

Answer 1

经过太多时间花在这上面我终于得到了它的工作:

这一行:

signature=`/bin/echo -n "$stringToSign" | openssl sha1 -hmac ${s3Secret} -binary | base64`

缺少'e':

signature=`/bin/echo -en "$stringToSign" | openssl sha1 -hmac ${s3Secret} -binary | base64`

换句话说,在字符串签名之前,字符没有被转义.

另外,我还了解到,对于获取请求,内容类型毫无意义.

Answer 2

使用此线程中的各种答案，我将其转换为一个方便的s3getbash 函数：

#!/bin/bash

#usage - s3get writes the specified object to stdout
#  s3get <bucket/key> [region]

#set these in your environment/profile (NOT HERE)
AWS_ACCESS_KEY="" 
AWS_SECRET_KEY=""

#example usage
s3get my-bucket/a/path/to/my/file > /tmp/file

function s3get {
    #helper functions
    function fail { echo "$1" > /dev/stderr; exit 1; }
    #dependency check
    if ! hash openssl 2>/dev/null; then fail "openssl not installed"; fi
    if ! hash curl 2>/dev/null; then fail "curl not installed"; fi
    #params
    path="${1}"
    bucket=$(cut -d '/' -f 1 <<< "$path")
    key=$(cut -d '/' -f 2- <<< "$path")
    region="${2:-us-west-1}"
    #load creds
    access="$AWS_ACCESS_KEY"
    secret="$AWS_SECRET_KEY"
    #validate
    if [[ "$bucket" = "" ]]; then fail "missing bucket (arg 1)"; fi;
    if [[ "$key" = ""    ]]; then fail "missing key (arg 1)"; fi;
    if [[ "$region" = "" ]]; then fail "missing region (arg 2)"; fi;
    if [[ "$access" = "" ]]; then fail "missing AWS_ACCESS_KEY (env var)"; fi;
    if [[ "$secret" = "" ]]; then fail "missing AWS_SECRET_KEY (env var)"; fi;
    #compute signature
    contentType="text/html; charset=UTF-8" 
    date="`date -u +'%a, %d %b %Y %H:%M:%S GMT'`"
    resource="/${bucket}/${key}"
    string="GET\n\n${contentType}\n\nx-amz-date:${date}\n${resource}"
    signature=`echo -en $string | openssl sha1 -hmac "${secret}" -binary | base64` 
    #get!
    curl -H "x-amz-date: ${date}" \
        -H "Content-Type: ${contentType}" \
        -H "Authorization: AWS ${access}:${signature}" \
        "https://s3-${region}.amazonaws.com${resource}"
}

在 OSX 和 Ubuntu 上测试。保存在这个Github gist 中。

Answer 3

TS 要求提供脚本的有效 SHA-1 版本。然而，SHA-1 已经过时，并且 Amazon 的数据中心仅接受 SHA-256 加密，因此下载脚本可用于所有 S3 数据中心：它还遵循 HTTP 307 重定向。

#!/bin/sh

#USAGE:
# download-aws.sh <bucket> <region> <source-file> <dest-file>

set -e

s3Key=xxxxxxxxxxxxxxxxxxxx
s3Secret=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

file=$3
bucket=$1
host="${bucket}.s3.amazonaws.com"
resource="/${file}"
contentType="text/plain"
dateValue="`date +'%Y%m%d'`"
X_amz_date="`date +'%Y%m%dT%H%M%SZ'`"
X_amz_algorithm="AWS4-HMAC-SHA256"
awsRegion=$2
awsService="s3"
X_amz_credential="$s3Key%2F$dateValue%2F$awsRegion%2F$awsService%2Faws4_request"
X_amz_credential_auth="$s3Key/$dateValue/$awsRegion/$awsService/aws4_request"

signedHeaders="host;x-amz-algorithm;x-amz-content-sha256;x-amz-credential;x-amz-date"
contentHash="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"

HMAC_SHA256_asckey () {
        var=`/bin/echo -en $2 | openssl sha256 -hmac $1 -binary | xxd -p -c256`
        echo $var
}
HMAC_SHA256 () {
        var=`/bin/echo -en $2 | openssl dgst -sha256 -mac HMAC -macopt hexkey:$1 -binary | xxd -p -c256`
        echo $var
}
REQUEST () {
        canonicalRequest="GET\n$resource\n\n"\
"host:$1\n"\
"x-amz-algorithm:$X_amz_algorithm""\n"\
"x-amz-content-sha256:$contentHash""\n"\
"x-amz-credential:$X_amz_credential""\n"\
"x-amz-date:$X_amz_date""\n\n"\
"$signedHeaders\n"\
"$contentHash"
        #echo $canonicalRequest
        canonicalHash=`/bin/echo -en "$canonicalRequest" | openssl sha256 -binary | xxd -p -c256`
        stringToSign="$X_amz_algorithm\n$X_amz_date\n$dateValue/$awsRegion/s3/aws4_request\n$canonicalHash"
        #echo $stringToSign


        s1=`HMAC_SHA256_asckey "AWS4""$s3Secret" $dateValue`
        s2=`HMAC_SHA256 "$s1" "$awsRegion"`
        s3=`HMAC_SHA256 "$s2" "$awsService"`
        signingKey=`HMAC_SHA256 "$s3" "aws4_request"`
        signature=`/bin/echo -en $stringToSign | openssl dgst -sha256 -mac HMAC -macopt hexkey:$signingKey -binary | xxd -p -c256`
        #echo signature

        authorization="$X_amz_algorithm Credential=$X_amz_credential_auth,SignedHeaders=$signedHeaders,Signature=$signature"
        result=$(curl --silent -H "Host: $1" -H "X-Amz-Algorithm: $X_amz_algorithm" -H "X-Amz-Content-Sha256: $contentHash" -H "X-Amz-Credential: $X_amz_credential" -H "X-Amz-Date: $X_amz_date" -H "Authorization: $authorization" https://${1}/${file} -o "$2" --write-out "%{http_code}")
        if [ $result -eq 307 ]; then
                redirecthost=`cat $2 | sed -n 's:.*<Endpoint>\(.*\)</Endpoint>.*:\1:p'`
                REQUEST "$redirecthost" "$2"
        fi
}
REQUEST "$host" "$4"

在 Ubuntu 上测试

如果有人知道删除 HMAC-ASCII 步骤的解决方案，欢迎您回复。我只能以这种方式工作。