diff --git a/.env.example b/.env.example index a78ff86..9850cac 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,9 @@ ## obs-access-signer # + +SERVER_MODE=s3 +# or SERVER_MODE=storj + HTTP_ADDR=127.0.0.1:9003 OBS_ENDPOINT=127.0.0.1:9000 OBS_BUCKET_NAME=test-bucket @@ -11,7 +15,10 @@ AWS_SECRET_KEY=example-minio-secret # accessible S3 gateway OBS_REDIRECT_SECURE=false +OBS_REDIRECT_CODE=307 OBS_HOST_REDIRECT=127.0.0.1:9000 # OBS_REDIRECT_CODE=307 # temporary redirect -# OBS_URL_EXPIRY=48h # 2 days \ No newline at end of file +# OBS_URL_EXPIRY=48h # 2 days + +# UPLINK_ACCESS_GRANT= # Storj Access Grant token \ No newline at end of file diff --git a/README.md b/README.md index 4d72031..9db9200 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ S3 Object Storage access signer. -Run `obs-access-signer` behind a gateway/cache proxy is preferred as the response is static. +Run `obs-access-signer` behind a gateway/cache proxy is preferred as the response is static (See "Why?" section). There's an example of using it with Varnish Cache, which you can see [here](docker/docker-compose.yaml). @@ -12,6 +12,8 @@ Some S3-compatible gateways might not support ACL endpoints but they support pre Update: starting at v0.0.3, obs-access-signer supports custom redirection status code and expiry. +Update: starting at v0.0.4, obs-access-signer supports `libuplink`, we can use this feature by specifying `SERVER_MODE=storj` on the environment variable OR `-server=storj` CLI flag. + ## License Apache-2.0 diff --git a/go.mod b/go.mod index cd66fe5..17e3aca 100644 --- a/go.mod +++ b/go.mod @@ -5,30 +5,46 @@ go 1.19 require ( github.com/joho/godotenv v1.4.0 github.com/minio/minio-go/v7 v7.0.45 + github.com/pkg/errors v0.8.1 + github.com/stretchr/testify v1.8.0 github.com/valyala/fasthttp v1.43.0 go.uber.org/zap v1.24.0 + storj.io/uplink v1.10.0 ) require ( github.com/andybalholm/brotli v1.0.4 // indirect + github.com/calebcase/tmpfile v1.0.3 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/google/uuid v1.3.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b // indirect github.com/klauspost/compress v1.15.9 // indirect github.com/klauspost/cpuid/v2 v2.1.0 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rs/xid v1.4.0 // indirect github.com/sirupsen/logrus v1.9.0 // indirect + github.com/spacemonkeygo/monkit/v3 v3.0.19 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect + github.com/zeebo/blake3 v0.2.3 // indirect + github.com/zeebo/errs v1.3.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/goleak v1.1.12 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect golang.org/x/net v0.0.0-20220906165146-f3363e06e74c // indirect + golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect golang.org/x/text v0.3.7 // indirect gopkg.in/ini.v1 v1.66.6 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + storj.io/common v0.0.0-20221123115229-fed3e6651b63 // indirect + storj.io/drpc v0.0.32 // indirect ) diff --git a/go.sum b/go.sum index d7884b6..592a6b0 100644 --- a/go.sum +++ b/go.sum @@ -1,26 +1,38 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/calebcase/tmpfile v1.0.3 h1:BZrOWZ79gJqQ3XbAQlihYZf/YCV0H4KPIdM5K5oMpJo= +github.com/calebcase/tmpfile v1.0.3/go.mod h1:UAUc01aHeC+pudPagY/lWvt2qS9ZO5Zzof6/tIUzqeI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20211108044417-e9b028704de0 h1:rsq1yB2xiFLDYYaYdlGBsSkwVzsCo500wMhxvW5A/bk= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b h1:tO4MX3k5bvV0Sjv5jYrxStMTJxf1m/TW24XRyHji4aU= +github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b/go.mod h1:q7yMR8BavTz/gBNtIT/uF487LMgcuEpNGKISLAjNQes= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0= github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= @@ -34,22 +46,41 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spacemonkeygo/monkit/v3 v3.0.19 h1:wqBb9bpD7jXkVi4XwIp8jn1fektaVBQ+cp9SHRXgAdo= +github.com/spacemonkeygo/monkit/v3 v3.0.19/go.mod h1:kj1ViJhlyADa7DiA4xVnTuPA46lFKbM7mxQTrXCuJP4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.43.0 h1:Gy4sb32C98fbzVWZlTM1oTMdLWGyvxR03VhM6cBIU4g= github.com/valyala/fasthttp v1.43.0/go.mod h1:f6VbjjoI3z1NDOZOv17o6RvtRSWxC77seBFc2uWtgiY= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 h1:zMsHhfK9+Wdl1F7sIKLyx3wrOFofpb3rWFbA4HgcK5k= +github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3/go.mod h1:R0Gbuw7ElaGSLOZUSwBm/GgVwMd30jWxBDdAyMOeTuc= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/assert v1.3.1 h1:vukIABvugfNMZMQO1ABsyQDJDTVQbn+LWSMy1ol1h6A= +github.com/zeebo/assert v1.3.1/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= +github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ= +github.com/zeebo/errs v1.3.0 h1:hmiaKqgYZzcVgRL1Vkc1Mn2914BbzB0IBxs+ebeutGs= +github.com/zeebo/errs v1.3.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= +github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= @@ -60,26 +91,39 @@ go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220906165146-f3363e06e74c h1:yKufUcDwucU5urd+50/Opbt4AYpqthk7wHpHok8f1lo= golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde h1:ejfdSekXMDxDLbRrJMwUk6KnSLZ2McaUCVcIKM+N6jc= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -95,13 +139,24 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI= gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +storj.io/common v0.0.0-20221123115229-fed3e6651b63 h1:OuleF/3FvZe3Nnu6NdwVr+FvCXjfD4iNNdgfI2kcs3k= +storj.io/common v0.0.0-20221123115229-fed3e6651b63/go.mod h1:+gF7jbVvpjVIVHhK+EJFhfPbudX395lnPq/dKkj/Qys= +storj.io/drpc v0.0.32 h1:5p5ZwsK/VOgapaCu+oxaPVwO6UwIs+iwdMiD50+R4PI= +storj.io/drpc v0.0.32/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg= +storj.io/uplink v1.10.0 h1:3hS0hszupHSxEoC4DsMpljaRy0uNoijEPVF6siIE28Q= +storj.io/uplink v1.10.0/go.mod h1:gJIQumB8T3tBHPRive51AVpbc+v2xe+P/goFNMSRLG4= diff --git a/main.go b/main.go index 09b063b..d092af6 100644 --- a/main.go +++ b/main.go @@ -1,10 +1,11 @@ package main import ( + "context" "flag" + "fmt" "os" - "strconv" - "time" + "strings" _ "github.com/joho/godotenv/autoload" "go.uber.org/zap" @@ -12,21 +13,62 @@ import ( ) var ( - httpAddr string - logLevel string + httpAddr = ":9003" + logLevel = "INFO" + serverMode = "s3" zapLogLevel zapcore.Level postFlagParse = []func(){} + + registeredServers = []Server{ + &serverS3{}, + &serverStorj{}, + } + mappedServers = map[string]Server{} ) func init() { var err error _ = err - // -- app - flag.StringVar(&httpAddr, "addr", os.Getenv("HTTP_ADDR"), "Server address") + /* --- [preload] --- */ + var availableServerNames []string + for _, s := range registeredServers { + serverName := s.Name() + if old, exist := mappedServers[serverName]; exist { + panic(fmt.Sprintf("duplicate server name old: %T, new: %T", old, s)) + } + mappedServers[serverName] = s + availableServerNames = append(availableServerNames, serverName) + } - // -- log - flag.StringVar(&logLevel, "log-level", os.Getenv("LOG_LEVEL"), "Log level") + /* --- app --- */ + var vHttpAddr = httpAddr + if sHttpAddr := os.Getenv("HTTP_ADDR"); sHttpAddr != "" { + vHttpAddr = sHttpAddr + } + flag.StringVar(&httpAddr, "addr", vHttpAddr, "Server address") + + var vServerMode = serverMode + if sServerMode := os.Getenv("SERVER_MODE"); sServerMode != "" { + vServerMode = sServerMode + } + flag.StringVar(&serverMode, "server", vServerMode, + fmt.Sprintf("Server mode (available [%s])", strings.Join(availableServerNames, ", "))) + qpostFlagParse(func() { + if httpAddr == "" { + httpAddr = ":9003" + } + if serverMode == "" { + serverMode = "s3" + } + }) + + /* --- log --- */ + var vLogLevel = logLevel + if sLogLevel := os.Getenv("LOG_LEVEL"); sLogLevel != "" { + vLogLevel = sLogLevel + } + flag.StringVar(&logLevel, "log-level", vLogLevel, "Log level") qpostFlagParse(func() { err := zapLogLevel.UnmarshalText([]byte(logLevel)) if err != nil { @@ -34,34 +76,20 @@ func init() { } }) - // -- OBS - flag.StringVar(&defaultObsOpts.Endpoint, "obs-endpoint", os.Getenv("OBS_ENDPOINT"), "OBS host") - flag.StringVar(&defaultObsOpts.Region, "obs-region", os.Getenv("OBS_REGION"), "OBS region") - flag.BoolVar(&defaultObsOpts.Secure, "obs-secure", ok1(strconv.ParseBool(os.Getenv("OBS_SECURE"))), "OBS secure transport") - flag.StringVar(&defaultObsOpts.BucketName, "obs-bucket", os.Getenv("OBS_BUCKET_NAME"), "OBS bucket name") - - flag.BoolVar(&defaultObsOpts.RedirectSecure, "obs-redirect-secure", ok1(strconv.ParseBool(os.Getenv("OBS_REDIRECT_SECURE"))), "OBS redirect secure transport") - flag.StringVar(&defaultObsOpts.HostRedirect, "obs-host-redirect", os.Getenv("OBS_HOST_REDIRECT"), "OBS host redirect") - - // redirect http code - var obsRedirectCode = int64(defaultObsOpts.RedirectCode) - if obsRedirectCodeStr := os.Getenv("OBS_REDIRECT_CODE"); obsRedirectCodeStr != "" { - obsRedirectCode, err = strconv.ParseInt(obsRedirectCodeStr, 10, 64) - if err != nil { - obsRedirectCode = int64(defaultObsOpts.RedirectCode) - } + /* --- OBS --- */ + if err = defaultObsOpts.Bind(flag.CommandLine); err != nil { + panic(err) } - flag.IntVar(&defaultObsOpts.RedirectCode, "obs-redirect-code", int(obsRedirectCode), "OBS redirect http code") - // url expiry - var obsUrlExpiry = defaultObsOpts.URLExpiry - if obsUrlExpiryStr := os.Getenv("OBS_URL_EXPIRY"); obsUrlExpiryStr != "" { - if obsUrlExpiry, err = time.ParseDuration(obsUrlExpiryStr); err != nil { - obsUrlExpiry = defaultObsOpts.URLExpiry - } + /* --- OBS S3 --- */ + if err = defaultObsS3Opts.Bind(flag.CommandLine); err != nil { + panic(err) } - flag.DurationVar(&defaultObsOpts.URLExpiry, "obs-url-expiry", obsUrlExpiry, "OBS url expiry") + /* --- OBS Storj (via LibUplink) --- */ + if err = defaultObsUplinkOpts.Bind(flag.CommandLine); err != nil { + panic(err) + } } func qpostFlagParse(f func()) { @@ -86,21 +114,34 @@ func main() { sug := logger.Named("main").Sugar() sug.Infow("starting", "log_level", zapLogLevel, + "server_mode", serverMode, + // Generic OBS "obs_bucket", defaultObsOpts.BucketName, - "obs_endpoint", defaultObsOpts.Endpoint, "obs_redirect_secure", defaultObsOpts.RedirectSecure, "obs_host_redirect", defaultObsOpts.HostRedirect, "obs_redirect_code", defaultObsOpts.RedirectCode, "obs_url_expiry", defaultObsOpts.URLExpiry.String(), + // S3 + "obs_s3_endpoint", defaultObsS3Opts.Endpoint, + // Storj (via LibUplink) + "obs_storj_satellite_addr", defaultObsUplinkOpts.SatelliteAddress, ) - client := unwrap1(newObsClient(defaultObsOpts)) - srv.Init(serverOptions{ - Addr: httpAddr, - Logger: logger.Named("server"), - OBS: &defaultObsOpts, - S3: client, - }) + // lookup server mode handler + srv, exist := mappedServers[serverMode] + if !exist || srv == nil { + sug.Fatalw("unknown server handler", + "server_mode", serverMode) + } - srv.Run() + // run http server + RunServer(context.Background(), + srv, + serverOptions{ + Addr: httpAddr, + Logger: logger.Named("server"), + Opts: &defaultObsOpts, + S3Opts: &defaultObsS3Opts, + UplinkOpts: &defaultObsUplinkOpts, + }) } diff --git a/obs.go b/obs.go index 544d9f2..fa72375 100644 --- a/obs.go +++ b/obs.go @@ -1,111 +1,68 @@ package main import ( - "context" - "io" + "flag" "net/http" - "net/url" - "reflect" + "os" + "strconv" "time" - "unsafe" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" - - _ "unsafe" + "github.com/pkg/errors" ) type obsOptions struct { - Endpoint string - Region string - Secure bool - BucketName string - + BucketName string RedirectSecure bool RedirectCode int // HTTP redirect status code URLExpiry time.Duration HostRedirect string } -const maxURLExpiry = time.Duration(int64(^uint64(0) / 2)) - var defaultObsOpts = obsOptions{ URLExpiry: maxURLExpiry, RedirectCode: http.StatusMovedPermanently, // 301 } -func newObsClient(opts obsOptions) (*minio.Client, error) { - client, err := minio.New(opts.Endpoint, &minio.Options{ - Creds: credentials.NewEnvAWS(), - BucketLookup: minio.BucketLookupAuto, // vhost / path - Region: opts.Region, - Secure: opts.Secure, - }) - if err != nil { - return nil, err +func (opts *obsOptions) Bind(fs *flag.FlagSet) (err error) { + + var vBucketName = opts.BucketName + if sBucketName := os.Getenv("OBS_BUCKET_NAME"); sBucketName != "" { + vBucketName = sBucketName } - setOverrideSignerType(client, credentials.SignatureV2) - return client, nil -} + fs.StringVar(&opts.BucketName, "obs-bucket", vBucketName, "OBS Bucket name") -var ( - offsetCredsProvider uintptr - offsetOverrideSignerType uintptr -) - -func init() { - vt := reflect.TypeOf(minio.Client{}) - if field, ok := vt.FieldByName("credsProvider"); ok { - offsetCredsProvider = field.Offset - } else { - panic("cannot find credsProvider field") + var vRedirectSecure = opts.RedirectSecure + if sRedirectSecure := os.Getenv("OBS_REDIRECT_SECURE"); sRedirectSecure != "" { + vRedirectSecure, _ = strconv.ParseBool(sRedirectSecure) } + fs.BoolVar(&opts.RedirectSecure, "obs-redirect-secure", vRedirectSecure, "OBS Redirect secure") - if field, ok := vt.FieldByName("overrideSignerType"); ok { - offsetOverrideSignerType = field.Offset - } else { - panic("cannot find overrideSignerType field") + var vObsHostRedirect = opts.HostRedirect + if sObsHostRedirect := os.Getenv("OBS_HOST_REDIRECT"); sObsHostRedirect != "" { + vObsHostRedirect = sObsHostRedirect } + fs.StringVar(&opts.HostRedirect, "obs-host-redirect", vObsHostRedirect, "OBS Host redirect") + + var vObsRedirectCode = opts.RedirectCode + if sObsRedirectCode := os.Getenv("OBS_REDIRECT_CODE"); sObsRedirectCode != "" { + var obsRedirectCode int64 + if obsRedirectCode, err = strconv.ParseInt(sObsRedirectCode, 10, 64); err != nil { + err = errors.Wrap(err, "obs redirect code") + return + } + vObsRedirectCode = int(obsRedirectCode) + } + fs.IntVar(&opts.RedirectCode, "obs-redirect-code", vObsRedirectCode, "OBS Redirect code") + + var vObsUrlExpiry = opts.URLExpiry + if sObsUrlExpiry := os.Getenv("OBS_URL_EXPIRY"); sObsUrlExpiry != "" { + var obsUrlExpiry time.Duration + if obsUrlExpiry, err = time.ParseDuration(sObsUrlExpiry); err != nil { + err = errors.Wrap(err, "obs url expiry") + return + } + vObsUrlExpiry = obsUrlExpiry + } + fs.DurationVar(&opts.URLExpiry, "obs-url-expiry", vObsUrlExpiry, "OBS Redirection URL expiry") + return } - -func getCredsProvider(client *minio.Client) *credentials.Credentials { - return *(**credentials.Credentials)(unsafe.Add(unsafe.Pointer(client), offsetCredsProvider)) -} - -func setOverrideSignerType(client *minio.Client, signerType credentials.SignatureType) { - ptr := (*credentials.SignatureType)(unsafe.Add(unsafe.Pointer(client), offsetOverrideSignerType)) - *ptr = signerType -} - -//go:linkname isVirtualHostStyleRequest github.com/minio/minio-go/v7.(*Client).isVirtualHostStyleRequest -func isVirtualHostStyleRequest(client *minio.Client, url url.URL, bucketName string) bool - -//go:linkname makeTargetURL github.com/minio/minio-go/v7.(*Client).makeTargetURL -func makeTargetURL(client *minio.Client, bucketName, objectName, bucketLocation string, isVirtualHostStyle bool, queryValues url.Values) (*url.URL, error) - -// requestMetadata - is container for all the values to make a request. -type requestMetadata struct { - // If set newRequest presigns the URL. - presignURL bool - - // User supplied. - bucketName string - objectName string - queryValues url.Values - customHeader http.Header - extraPresignHeader http.Header - expires int64 - - // Generated by our internal code. - bucketLocation string - contentBody io.Reader - contentLength int64 - contentMD5Base64 string // carries base64 encoded md5sum - contentSHA256Hex string // carries hex encoded sha256sum - streamSha256 bool - addCrc bool - trailer http.Header // (http.Request).Trailer. Requires v4 signature. -} - -//go:linkname newRequest github.com/minio/minio-go/v7.(*Client).newRequest -func newRequest(client *minio.Client, ctx context.Context, method string, metadata requestMetadata) (req *http.Request, err error) diff --git a/obs_s3.go b/obs_s3.go new file mode 100644 index 0000000..d5b5428 --- /dev/null +++ b/obs_s3.go @@ -0,0 +1,124 @@ +package main + +import ( + "context" + "flag" + "io" + "net/http" + "net/url" + "os" + "reflect" + "strconv" + "time" + "unsafe" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/pkg/errors" + + _ "unsafe" +) + +type obsS3Options struct { + Endpoint string + Region string + Secure bool // S3 secure +} + +const maxURLExpiry = time.Duration(int64(^uint64(0) / 2)) + +var defaultObsS3Opts = obsS3Options{} + +func (opts *obsS3Options) Bind(fs *flag.FlagSet) (err error) { + fs.StringVar(&opts.Endpoint, "obs-endpoint", os.Getenv("OBS_ENDPOINT"), "OBS S3 Host") + fs.StringVar(&opts.Region, "obs-region", os.Getenv("OBS_REGION"), "OBS S3 Region") + + var vObsSecure = opts.Secure + if sObsSecure := os.Getenv("OBS_SECURE"); sObsSecure != "" { + var obsSecure bool + if obsSecure, err = strconv.ParseBool(sObsSecure); err != nil { + err = errors.Wrap(err, "obs secure") + return + } + vObsSecure = obsSecure + } + fs.BoolVar(&opts.Secure, "obs-secure", vObsSecure, "OBS S3 secure transport") + + return +} + +func newObsS3Client(opts obsS3Options) (*minio.Client, error) { + client, err := minio.New(opts.Endpoint, &minio.Options{ + Creds: credentials.NewEnvAWS(), + BucketLookup: minio.BucketLookupAuto, // vhost / path + Region: opts.Region, + Secure: opts.Secure, + }) + if err != nil { + return nil, err + } + setOverrideSignerType(client, credentials.SignatureV2) + return client, nil +} + +var ( + offsetCredsProvider uintptr + offsetOverrideSignerType uintptr +) + +func init() { + vt := reflect.TypeOf(minio.Client{}) + if field, ok := vt.FieldByName("credsProvider"); ok { + offsetCredsProvider = field.Offset + } else { + panic("cannot find credsProvider field") + } + + if field, ok := vt.FieldByName("overrideSignerType"); ok { + offsetOverrideSignerType = field.Offset + } else { + panic("cannot find overrideSignerType field") + } +} + +func getCredsProvider(client *minio.Client) *credentials.Credentials { + return *(**credentials.Credentials)(unsafe.Add(unsafe.Pointer(client), offsetCredsProvider)) +} + +func setOverrideSignerType(client *minio.Client, signerType credentials.SignatureType) { + ptr := (*credentials.SignatureType)(unsafe.Add(unsafe.Pointer(client), offsetOverrideSignerType)) + *ptr = signerType +} + +//go:linkname isVirtualHostStyleRequest github.com/minio/minio-go/v7.(*Client).isVirtualHostStyleRequest +func isVirtualHostStyleRequest(client *minio.Client, url url.URL, bucketName string) bool + +//go:linkname makeTargetURL github.com/minio/minio-go/v7.(*Client).makeTargetURL +func makeTargetURL(client *minio.Client, bucketName, objectName, bucketLocation string, isVirtualHostStyle bool, queryValues url.Values) (*url.URL, error) + +// requestMetadata - is container for all the values to make a request. +type requestMetadata struct { + // If set newRequest presigns the URL. + presignURL bool + + // User supplied. + bucketName string + objectName string + queryValues url.Values + customHeader http.Header + extraPresignHeader http.Header + expires int64 + + // Generated by our internal code. + bucketLocation string + contentBody io.Reader + contentLength int64 + contentMD5Base64 string // carries base64 encoded md5sum + contentSHA256Hex string // carries hex encoded sha256sum + streamSha256 bool + addCrc bool + trailer http.Header // (http.Request).Trailer. Requires v4 signature. +} + +//go:linkname newRequest github.com/minio/minio-go/v7.(*Client).newRequest +func newRequest(client *minio.Client, ctx context.Context, method string, metadata requestMetadata) (req *http.Request, err error) diff --git a/obs_test.go b/obs_s3_test.go similarity index 100% rename from obs_test.go rename to obs_s3_test.go diff --git a/obs_storj.go b/obs_storj.go new file mode 100644 index 0000000..fd23343 --- /dev/null +++ b/obs_storj.go @@ -0,0 +1,179 @@ +package main + +import ( + "context" + "flag" + "os" + + "github.com/pkg/errors" + "storj.io/uplink" + "storj.io/uplink/edge" +) + +type obsStorjOptions struct { + SatelliteAddress string + APIKey string + Passphrase string + + AccessGrant string + + AccessKeyID string + ShareBaseURL string +} + +var defaultObsUplinkOpts = obsStorjOptions{ + // we can override satellite address from Access Grant with this + // ex. "ap1.storj.io:7777" + SatelliteAddress: "", + ShareBaseURL: "https://link.storjshare.io", +} + +func (opts *obsStorjOptions) Bind(fs *flag.FlagSet) (err error) { + var vSatelliteAddr = opts.SatelliteAddress + if sSatelliteAddr := os.Getenv("UPLINK_SATELLITE_ADDR"); sSatelliteAddr != "" { + vSatelliteAddr = sSatelliteAddr + } + fs.StringVar(&opts.SatelliteAddress, "uplink-satellite-addr", vSatelliteAddr, "OBS Storj Satellite Address") + + { + var vAPIKey = opts.APIKey + if sAPIKey := os.Getenv("UPLINK_API_KEY"); sAPIKey != "" { + vAPIKey = sAPIKey + } + fs.StringVar(&opts.APIKey, "uplink-api-key", vAPIKey, "OBS Storj API key") + + var vPassphrase = opts.Passphrase + if sPassphrase := os.Getenv("UPLINK_PASSPHRASE"); sPassphrase != "" { + vPassphrase = sPassphrase + } + fs.StringVar(&opts.Passphrase, "uplink-passphrase", vPassphrase, "OBS Storj Passphrase") + } + + var vAccessGrant = opts.AccessGrant + if sAccessGrant := os.Getenv("UPLINK_ACCESS_GRANT"); sAccessGrant != "" { + vAccessGrant = sAccessGrant + } + fs.StringVar(&opts.AccessGrant, "uplink-access-grant", vAccessGrant, "OBS Storj Access Grant") + + var vAccessKeyID = opts.AccessKeyID + if sAccessKeyID := os.Getenv("UPLINK_ACCESS_KEY_ID"); sAccessKeyID != "" { + vAccessKeyID = sAccessKeyID + } + fs.StringVar(&opts.AccessKeyID, "uplink-access-key-id", vAccessKeyID, "OBS Storj Access key ID") + + var vShareBaseURL = opts.ShareBaseURL + if sShareBaseURL := os.Getenv("UPLINK_SHARE_BASE_URL"); sShareBaseURL != "" { + vShareBaseURL = sShareBaseURL + } + fs.StringVar(&opts.ShareBaseURL, "uplink-share-base-url", vShareBaseURL, "OBS Storj Link Share base URL") + + return +} + +var defaultEdgeConfig = edge.Config{ + AuthServiceAddress: "auth.storjshare.io:7777", +} + +type storjAggegrateClient struct { + edgeConfig *edge.Config + access *uplink.Access + project *uplink.Project + + creds *edge.Credentials + + accessKeyID string + shareBaseURL string +} + +func (c *storjAggegrateClient) Init(ctx context.Context) (_ *storjAggegrateClient, err error) { + if c.access != nil { + // TODO: rate limit this, persist the registered access key ID + if c.creds, err = c.edgeConfig.RegisterAccess(ctx, c.access, &edge.RegisterAccessOptions{ + // This will create `accessKeyID` that allow anonymous access + // to any objects that `c.access` has access to (just like `public-read` ACL). + Public: true, + }); err != nil { + err = errors.Wrap(err, "register access") + return + } + } + return c, nil +} + +func (c *storjAggegrateClient) getAccessKeyID() (accessKeyID string) { + accessKeyID = c.accessKeyID + // fallback to `c.creds` if custom accessKeyID is not provided + // and `c.creds` is specified. + if accessKeyID == "" && c.creds != nil { + accessKeyID = c.creds.AccessKeyID + } + return +} + +func (c *storjAggegrateClient) getProject() *uplink.Project { + return c.project +} + +func (c *storjAggegrateClient) JoinShareURL(bucket, key string, opts *edge.ShareURLOptions) (string, error) { + accessKeyID := c.getAccessKeyID() + return edge.JoinShareURL(c.shareBaseURL, accessKeyID, bucket, key, opts) +} + +func newObsStorjClient(ctx context.Context, opts obsStorjOptions) (client *storjAggegrateClient, err error) { + var ( + access *uplink.Access + project *uplink.Project + ) + switch { + case opts.AccessGrant != "": + // Docs: https://docs.storj.io/dcs/concepts/access/access-grants + access, err = uplink.ParseAccess(opts.AccessGrant) + if err != nil { + err = errors.Wrap(err, "parse access") + return + } + case opts.APIKey != "" && opts.Passphrase != "": + // Docs: https://docs.storj.io/dcs/getting-started/quickstart-uplink-cli/generate-access-grants-and-tokens/generate-a-token + if access, err = uplink.RequestAccessWithPassphrase( + ctx, + opts.SatelliteAddress, + opts.APIKey, + opts.Passphrase, + ); err != nil { + err = errors.Wrap(err, "access with passphrase") + return + } + default: + // using custom accessKeyID + access = nil + } + // TODO: limit access scope. + if access != nil { + // Consider to limit the access to specific bucket and prefix + // access, err = access.Share( + // uplink.ReadOnlyPermission(), + // uplink.SharePrefix{ + // Bucket: "", + // Prefix: "", + // }) + + // open project + project, err = uplink.OpenProject(ctx, access) + if err != nil { + err = errors.Wrap(err, "open project") + return + } + } + // fallback default link sharing base url + if opts.ShareBaseURL == "" { + opts.ShareBaseURL = defaultObsUplinkOpts.ShareBaseURL + } + return (&storjAggegrateClient{ + edgeConfig: &defaultEdgeConfig, + access: access, + project: project, + + accessKeyID: opts.AccessKeyID, + shareBaseURL: opts.ShareBaseURL, + }).Init(ctx) +} diff --git a/obs_storj_test.go b/obs_storj_test.go new file mode 100644 index 0000000..b51d416 --- /dev/null +++ b/obs_storj_test.go @@ -0,0 +1,42 @@ +package main + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "storj.io/uplink/edge" +) + +func TestStorjUplink(t *testing.T) { + client, err := newObsStorjClient(context.Background(), obsStorjOptions{ + // Docs: https://docs.storj.io/dcs/getting-started/quickstart-uplink-cli/generate-access-grants-and-tokens/generate-a-token + // SatelliteAddress: "link.storjshare.io", + // APIKey: "xxx", + // Passphrase: "xxxx", + // --- or --- + // Docs: https://docs.storj.io/dcs/concepts/access/access-grants + // AccessGrant: "xxxx", + // --- or --- + // Docs: https://pkg.go.dev/storj.io/uplink/edge#Config.RegisterAccess + // > RegisterAccess gets credentials for the Storj-hosted Gateway and linkshare service. + // > All files accessible under the Access are then also accessible via those services. + // > If you call this function a lot, and the use case allows it, please limit + // > the lifetime of the credentials by setting Permission.NotAfter when creating the Access. + AccessKeyID: "placeholder", + }) + require.NoError(t, err) + + if client.access != nil { + println(client.access.SatelliteAddress()) + } + + shareLinkURL, err := client.JoinShareURL( + // "demo-bucket", "main.c", + "moe", "moe-onl/13744453-430b-4e6b-ae81-29e7f2491317.png", + &edge.ShareURLOptions{ + Raw: true, + }) + require.NoError(t, err) + println(shareLinkURL) +} diff --git a/server.go b/server.go index 581ab1a..10cdc14 100644 --- a/server.go +++ b/server.go @@ -1,20 +1,18 @@ package main import ( - "bytes" - "fmt" + "context" "net/http" - "net/url" - "strconv" - "time" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/s3utils" - "github.com/minio/minio-go/v7/pkg/signer" "github.com/valyala/fasthttp" "go.uber.org/zap" ) +var ( + ErrKind_ResourceNotFound = "OBS_RESOURCE_NOT_FOUND" + ErrKind_MethodNotAllowed = "OBS_METHOD_NOT_ALLOWED" +) + var ( MethodGet = []byte(http.MethodGet) MethodHead = []byte(http.MethodHead) @@ -23,25 +21,37 @@ var ( type serverOptions struct { Addr string Logger *zap.Logger - OBS *obsOptions - S3 *minio.Client + Opts *obsOptions + S3Opts *obsS3Options + UplinkOpts *obsStorjOptions } -type server struct { - opts serverOptions - logger *zap.SugaredLogger +func (s *serverOptions) GetOpts() obsOptions { + if s.Opts == nil { + return defaultObsOpts + } + return *s.Opts } -var srv server - -func (s *server) Init(opts serverOptions) { - s.opts = opts - s.logger = opts.Logger.Sugar() +func (s *serverOptions) GetS3Opts() obsS3Options { + if s.S3Opts == nil { + return defaultObsS3Opts + } + return *s.S3Opts } -func (s *server) reportError(ctx *fasthttp.RequestCtx, errType string, err any) { - s.logger.Errorw("handler error", +func (s *serverOptions) GetUplinkOpts() obsStorjOptions { + if s.UplinkOpts == nil { + return defaultObsUplinkOpts + } + return *s.UplinkOpts +} + +func reportError(self interface { + getLogger() *zap.SugaredLogger +}, ctx *fasthttp.RequestCtx, errType string, err any) { + self.getLogger().Errorw("handler error", "kind", errType, "err", err) ctx.Response.Header.Set("x-error-code", errType) @@ -57,115 +67,20 @@ func (s *server) reportError(ctx *fasthttp.RequestCtx, errType string, err any) } } -var ( - ErrKind_ResourceNotFound = "OBS_RESOURCE_NOT_FOUND" - ErrKind_MethodNotAllowed = "OBS_METHOD_NOT_ALLOWED" - ErrKind_S3ComposeRequest = "S3_COMPOSE_REQUEST" - ErrKind_S3CredsProvider = "S3_CREDS_PROVIDER" -) +type Server interface { + Init(ctx context.Context, opts serverOptions) (err error) + Name() string + GetHandler() fasthttp.RequestHandler +} -func (s *server) handle(ctx *fasthttp.RequestCtx) { - ctx.Response.Header.Set("server", "obs-access-signer") - isMethodGet := bytes.Equal(ctx.Method(), MethodGet) - isMethodHead := bytes.Equal(ctx.Method(), MethodHead) - if !isMethodGet && !isMethodHead { - ctx.SetStatusCode(http.StatusMethodNotAllowed) - s.reportError(ctx, ErrKind_MethodNotAllowed, "") - return - } - - if isMethodHead { - // Doc: https://www.rfc-editor.org/rfc/rfc9110.html#section-9.3.2-1 - ctx.Response.Header.Set("Content-Length", "0") - } - - bucketName := s.opts.OBS.BucketName - isVirtualHostStyle := isVirtualHostStyleRequest(s.opts.S3, *s.opts.S3.EndpointURL(), bucketName) - - path := ctx.Path() - _path := bytes.TrimLeft(path, "/") - objectName := unsafeByteSliceToString(_path) - - s.logger.Debugw("handle", - "bucket", bucketName, - "objectName", objectName) - - // check if we had access to the object - if _, err := s.opts.S3.StatObject(ctx, bucketName, objectName, minio.GetObjectOptions{}); err != nil { - ctx.SetStatusCode(http.StatusNotFound) - s.reportError(ctx, ErrKind_ResourceNotFound, err) - return - } - - // compose initial request - expireSeconds := int64(s.opts.OBS.URLExpiry / time.Second) - req, err := newRequest(s.opts.S3, ctx, http.MethodGet, requestMetadata{ - presignURL: true, - bucketName: bucketName, - objectName: objectName, - expires: expireSeconds, // to trigger presigned generator - queryValues: url.Values{}, +func RunServer(ctx context.Context, s Server, opts serverOptions) { + sug := opts.Logger.Sugar() + s.Init(ctx, opts) + sug.Infow("running server", + "addr", opts.Addr) + handler := s.GetHandler() + fasthttp.ListenAndServe(opts.Addr, func(ctx *fasthttp.RequestCtx) { + ctx.Response.Header.Set("server", "obs-access-signer") + handler(ctx) }) - if err != nil { - ctx.SetStatusCode(http.StatusInternalServerError) - s.reportError(ctx, ErrKind_S3ComposeRequest, err) - return - } - - // grab creds from provider - value, err := getCredsProvider(s.opts.S3).Get() - if err != nil { - ctx.SetStatusCode(http.StatusInternalServerError) - s.reportError(ctx, ErrKind_S3CredsProvider, err) - return - } - - var statusCode = s.opts.OBS.RedirectCode - - // custom "expiry" - var exp string - if expiry := s.opts.OBS.URLExpiry; expiry == maxURLExpiry || expiry <= 0 { - // clear given params, set max signed value for expire, and re-presign. - exp = strconv.FormatInt(int64(^uint64(0)/2), 10) // ~250years - } else { - // we can't allow a permanent redirect here since we already have - // expiry set, the redirected url needs to be updated. - if statusCode == http.StatusMovedPermanently || (statusCode < 300 || statusCode > 399) { - statusCode = http.StatusTemporaryRedirect - } - - expireAt := time.Now().UTC().Add(s.opts.OBS.URLExpiry) - exp = strconv.FormatInt(int64(expireAt.Unix()), 10) - // set object cache lifetime - if statusCode == http.StatusTemporaryRedirect { - ctx.Response.Header.Set("Cache-Control", fmt.Sprintf("max-age=%d", expireSeconds)) - ctx.Response.Header.Set("Expires", expireAt.Format("Mon, 02 Jan 2006 15:04:05 GMT")) - } - } - req.Header.Set("Expires", exp) - req.URL.RawQuery = "" - req = signer.PreSignV2(*req, value.AccessKeyID, value.SecretAccessKey, 0, isVirtualHostStyle) - - // re-encode query string with Expires hack. - query := req.URL.Query() - query.Set("Expires", exp) - req.URL.RawQuery = s3utils.QueryEncode(query) - - if s.opts.OBS.RedirectSecure { - req.URL.Scheme = "https" - } else { - req.URL.Scheme = "http" - } - - if hostRedirect := s.opts.OBS.HostRedirect; hostRedirect != "" { - req.URL.Host = hostRedirect - } - - ctx.Redirect(req.URL.String(), statusCode) -} - -func (s *server) Run() { - s.logger.Infow("running server", - "addr", s.opts.Addr) - fasthttp.ListenAndServe(s.opts.Addr, s.handle) } diff --git a/server_s3.go b/server_s3.go new file mode 100644 index 0000000..be8d46e --- /dev/null +++ b/server_s3.go @@ -0,0 +1,164 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/signer" + "github.com/pkg/errors" + "github.com/valyala/fasthttp" + "go.uber.org/zap" +) + +type serverS3 struct { + opts obsOptions + s3opts obsS3Options + + logger *zap.SugaredLogger + + s3c *minio.Client +} + +func (s *serverS3) Init(ctx context.Context, opts serverOptions) (err error) { + s.opts = opts.GetOpts() + s.s3opts = opts.GetS3Opts() + + s.logger = opts.Logger.Named(s.Name()).Sugar() + + if s.s3c, err = newObsS3Client(s.s3opts); err != nil { + err = errors.Wrap(err, "obs s3 client") + return + } + + return +} + +func (s *serverS3) Name() string { + return "s3" +} + +func (s *serverS3) getLogger() *zap.SugaredLogger { return s.logger } +func (s *serverS3) reportError(ctx *fasthttp.RequestCtx, errType string, err any) { + reportError(s, ctx, errType, err) +} + +var ( + ErrKind_S3ComposeRequest = "S3_COMPOSE_REQUEST" + ErrKind_S3CredsProvider = "S3_CREDS_PROVIDER" +) + +func (s *serverS3) handle(ctx *fasthttp.RequestCtx) { + isMethodGet := bytes.Equal(ctx.Method(), MethodGet) + isMethodHead := bytes.Equal(ctx.Method(), MethodHead) + if !isMethodGet && !isMethodHead { + ctx.SetStatusCode(http.StatusMethodNotAllowed) + s.reportError(ctx, ErrKind_MethodNotAllowed, "") + return + } + + if isMethodHead { + // Doc: https://www.rfc-editor.org/rfc/rfc9110.html#section-9.3.2-1 + ctx.Response.Header.Set("Content-Length", "0") + } + + bucketName := s.opts.BucketName + isVirtualHostStyle := isVirtualHostStyleRequest(s.s3c, *s.s3c.EndpointURL(), bucketName) + + path := ctx.Path() + _path := bytes.TrimLeft(path, "/") + if _, _pathWithoutBucketName, found := bytes.Cut(_path, []byte(`/`)); found { + // no need to check `isVirtualHostStyle` since this is our own implementation of handling request URI + _path = _pathWithoutBucketName + } + objectName := unsafeByteSliceToString(_path) + + s.logger.Debugw("handle", + "bucket", bucketName, + "objectName", objectName) + + // check if we had access to the object + if meta, err := s.s3c.StatObject(ctx, bucketName, objectName, minio.GetObjectOptions{}); err != nil { + ctx.SetStatusCode(http.StatusNotFound) + s.reportError(ctx, ErrKind_ResourceNotFound, err) + return + } else { + _ = meta + } + + // compose initial request + expireSeconds := int64(s.opts.URLExpiry / time.Second) + req, err := newRequest(s.s3c, ctx, http.MethodGet, requestMetadata{ + presignURL: true, + bucketName: bucketName, + objectName: objectName, + expires: expireSeconds, // to trigger presigned generator + queryValues: url.Values{}, + }) + if err != nil { + ctx.SetStatusCode(http.StatusInternalServerError) + s.reportError(ctx, ErrKind_S3ComposeRequest, err) + return + } + + // grab creds from provider + value, err := getCredsProvider(s.s3c).Get() + if err != nil { + ctx.SetStatusCode(http.StatusInternalServerError) + s.reportError(ctx, ErrKind_S3CredsProvider, err) + return + } + + var statusCode = s.opts.RedirectCode + + // custom "expiry" + var exp string + if expiry := s.opts.URLExpiry; expiry == maxURLExpiry || expiry <= 0 { + // clear given params, set max signed value for expire, and re-presign. + exp = strconv.FormatInt(int64(^uint64(0)/2), 10) // ~250years + } else { + // we can't allow a permanent redirect here since we already have + // expiry set, the redirected url needs to be updated. + if statusCode == http.StatusMovedPermanently || (statusCode < 300 || statusCode > 399) { + statusCode = http.StatusTemporaryRedirect + } + + expireAt := time.Now().UTC().Add(s.opts.URLExpiry) + exp = strconv.FormatInt(int64(expireAt.Unix()), 10) + // set redirect cache lifetime + if statusCode == http.StatusTemporaryRedirect { + ctx.Response.Header.Set("Cache-Control", fmt.Sprintf("max-age=%d", expireSeconds)) + ctx.Response.Header.Set("Expires", expireAt.Format("Mon, 02 Jan 2006 15:04:05 GMT")) + } + } + req.Header.Set("Expires", exp) + req.URL.RawQuery = "" + req = signer.PreSignV2(*req, value.AccessKeyID, value.SecretAccessKey, 0, isVirtualHostStyle) + + // re-encode query string with Expires hack. + query := req.URL.Query() + query.Set("Expires", exp) + req.URL.RawQuery = s3utils.QueryEncode(query) + + if s.opts.RedirectSecure { + req.URL.Scheme = "https" + } else { + req.URL.Scheme = "http" + } + + if hostRedirect := s.opts.HostRedirect; hostRedirect != "" { + req.URL.Host = hostRedirect + } + + ctx.Redirect(req.URL.String(), statusCode) +} + +func (s *serverS3) GetHandler() fasthttp.RequestHandler { + return s.handle +} diff --git a/server_storj.go b/server_storj.go new file mode 100644 index 0000000..225f789 --- /dev/null +++ b/server_storj.go @@ -0,0 +1,115 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "net/http" + "time" + + "github.com/pkg/errors" + "github.com/valyala/fasthttp" + "go.uber.org/zap" + "storj.io/uplink/edge" +) + +type serverStorj struct { + opts obsOptions + logger *zap.SugaredLogger + + sc *storjAggegrateClient +} + +func (s *serverStorj) Init(ctx context.Context, opts serverOptions) (err error) { + s.opts = opts.GetOpts() + s.logger = opts.Logger.Named(s.Name()).Sugar() + { + if s.sc, err = newObsStorjClient(ctx, opts.GetUplinkOpts()); err != nil { + err = errors.Wrap(err, "obs uplink client") + return + } + } + return +} + +func (s *serverStorj) Name() string { + return "storj" +} + +func (s *serverStorj) getLogger() *zap.SugaredLogger { return s.logger } +func (s *serverStorj) reportError(ctx *fasthttp.RequestCtx, errType string, err any) { + reportError(s, ctx, errType, err) +} + +var ( + ErrKind_StorjComposeShareURL = "STORJ_COMPOSE_SHARE_URL" +) + +func (s *serverStorj) handle(ctx *fasthttp.RequestCtx) { + isMethodGet := bytes.Equal(ctx.Method(), MethodGet) + isMethodHead := bytes.Equal(ctx.Method(), MethodHead) + if !(isMethodGet || isMethodHead) { + ctx.SetStatusCode(http.StatusMethodNotAllowed) + s.reportError(ctx, ErrKind_MethodNotAllowed, "") + return + } + + if isMethodHead { + // Doc: https://www.rfc-editor.org/rfc/rfc9110.html#section-9.3.2-1 + ctx.Response.Header.Set("Content-Length", "0") + } + + bucketName := s.opts.BucketName + path := ctx.Path() + _path := bytes.TrimLeft(path, "/") + if _, _pathWithoutBucketName, found := bytes.Cut(_path, []byte(`/`)); found { + // no need to check `isVirtualHostStyle` since this is our own implementation of handling request URI + _path = _pathWithoutBucketName + } + objectName := unsafeByteSliceToString(_path) + + s.logger.Debugw("handle", + "bucket", bucketName, + "objectName", objectName) + + // use project + if project := s.sc.getProject(); project != nil { + // check if we had access to the object + if meta, err := project.StatObject(ctx, bucketName, objectName); err != nil { + ctx.SetStatusCode(http.StatusNotFound) + s.reportError(ctx, ErrKind_ResourceNotFound, err) + return + } else { + _ = meta + } + } + + shareURL, err := s.sc.JoinShareURL(bucketName, objectName, &edge.ShareURLOptions{ + Raw: true, + }) + if err != nil { + ctx.SetStatusCode(http.StatusInternalServerError) + s.reportError(ctx, ErrKind_StorjComposeShareURL, err) + } + + var statusCode = s.opts.RedirectCode + + if statusCode < 300 || statusCode >= 399 { + // fallback of invalid redirect code + statusCode = http.StatusTemporaryRedirect + } + + expireAt := time.Now().UTC().Add(s.opts.URLExpiry) + expireSeconds := int64(s.opts.URLExpiry / time.Second) + // set redirect cache lifetime + if statusCode == http.StatusTemporaryRedirect { + ctx.Response.Header.Set("Cache-Control", fmt.Sprintf("max-age=%d", expireSeconds)) + ctx.Response.Header.Set("Expires", expireAt.Format("Mon, 02 Jan 2006 15:04:05 GMT")) + } + + ctx.Redirect(shareURL, s.opts.RedirectCode) +} + +func (s *serverStorj) GetHandler() fasthttp.RequestHandler { + return s.handle +}