Compare commits
1 commit
| Author | SHA1 | Date | |
|---|---|---|---|
9b66abc101
|
17 changed files with 223 additions and 938 deletions
|
|
@ -10,7 +10,6 @@ AWS_SECRET_KEY=example-minio-secret
|
|||
# AWS_SESSION_TOKEN
|
||||
|
||||
# accessible S3 gateway
|
||||
OBS_REDIRECT_SECURE=false
|
||||
OBS_HOST_REDIRECT=127.0.0.1:9000
|
||||
|
||||
|
||||
|
|
|
|||
11
.env.example
11
.env.example
|
|
@ -1,9 +1,5 @@
|
|||
## obs-access-signer
|
||||
#
|
||||
|
||||
SERVER_MODE=s3
|
||||
# or SERVER_MODE=storj
|
||||
|
||||
HTTP_ADDR=127.0.0.1:9003
|
||||
OBS_ENDPOINT=127.0.0.1:9000
|
||||
OBS_BUCKET_NAME=test-bucket
|
||||
|
|
@ -14,11 +10,4 @@ AWS_SECRET_KEY=example-minio-secret
|
|||
# AWS_SESSION_TOKEN
|
||||
|
||||
# accessible S3 gateway
|
||||
OBS_REDIRECT_SECURE=false
|
||||
OBS_REDIRECT_CODE=307
|
||||
OBS_HOST_REDIRECT=127.0.0.1:9000
|
||||
|
||||
# OBS_REDIRECT_CODE=307 # temporary redirect
|
||||
# OBS_URL_EXPIRY=48h # 2 days
|
||||
|
||||
# UPLINK_ACCESS_GRANT= # Storj Access Grant token
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -1,2 +1 @@
|
|||
.env
|
||||
obs-access-signer
|
||||
|
|
@ -1,27 +1,3 @@
|
|||
builds:
|
||||
- id: default
|
||||
env:
|
||||
- CGO_ENABLED=0
|
||||
goos:
|
||||
- linux
|
||||
- darwin
|
||||
ignore:
|
||||
- goos: darwin
|
||||
goarch: "386"
|
||||
- goos: linux
|
||||
goarch: "386"
|
||||
- id: windows
|
||||
env:
|
||||
- CGO_ENABLED=0
|
||||
goos:
|
||||
- windows
|
||||
ignore:
|
||||
- goos: windows
|
||||
goarch: "386"
|
||||
|
||||
release:
|
||||
prerelease: auto
|
||||
|
||||
dockers:
|
||||
- goos: linux
|
||||
goarch: amd64
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
S3 Object Storage access signer.
|
||||
|
||||
Run `obs-access-signer` behind a gateway/cache proxy is preferred as the response is static (See "Why?" section).
|
||||
Run `obs-access-signer` behind a gateway/cache proxy is preferred as the response is static.
|
||||
|
||||
There's an example of using it with Varnish Cache, which you can see [here](docker/docker-compose.yaml).
|
||||
|
||||
|
|
@ -10,10 +10,6 @@ There's an example of using it with Varnish Cache, which you can see [here](dock
|
|||
|
||||
Some S3-compatible gateways might not support ACL endpoints but they support presigned access. Currently, the behavior of `obs-access-signer` is similar to `public-read` ACL where clients can access objects anonymously and redirect them (permanently) to presigned URL with `Expires` set to the max signed value of `int64` which has roughly 250yrs lifetime since UNIX time started.
|
||||
|
||||
Update: starting at v0.0.3, obs-access-signer supports custom redirection status code and expiry.
|
||||
|
||||
Update: starting at v0.0.4, obs-access-signer supports `libuplink`, we can use this feature by specifying `SERVER_MODE=storj` on the environment variable OR `-server=storj` CLI flag.
|
||||
|
||||
## License
|
||||
|
||||
Apache-2.0
|
||||
|
|
|
|||
|
|
@ -3,11 +3,11 @@ version: '3'
|
|||
services:
|
||||
# obs access signer
|
||||
obs-access-signer:
|
||||
# image: obs-access-signer:dev
|
||||
build:
|
||||
context: ..
|
||||
image: obs-access-signer:dev
|
||||
networks:
|
||||
- obs
|
||||
build:
|
||||
context: ..
|
||||
env_file:
|
||||
- ../.config/example.env
|
||||
ports:
|
||||
|
|
|
|||
16
go.mod
16
go.mod
|
|
@ -5,46 +5,30 @@ go 1.19
|
|||
require (
|
||||
github.com/joho/godotenv v1.4.0
|
||||
github.com/minio/minio-go/v7 v7.0.45
|
||||
github.com/pkg/errors v0.8.1
|
||||
github.com/stretchr/testify v1.8.0
|
||||
github.com/valyala/fasthttp v1.43.0
|
||||
go.uber.org/zap v1.24.0
|
||||
storj.io/uplink v1.10.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.0.4 // indirect
|
||||
github.com/calebcase/tmpfile v1.0.3 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dustin/go-humanize v1.0.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b // indirect
|
||||
github.com/klauspost/compress v1.15.9 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.1.0 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/minio/sha256-simd v1.0.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/rs/xid v1.4.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.19 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect
|
||||
github.com/zeebo/blake3 v0.2.3 // indirect
|
||||
github.com/zeebo/errs v1.3.0 // indirect
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
go.uber.org/goleak v1.1.12 // indirect
|
||||
go.uber.org/multierr v1.6.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
|
||||
golang.org/x/net v0.0.0-20220906165146-f3363e06e74c // indirect
|
||||
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect
|
||||
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
gopkg.in/ini.v1 v1.66.6 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
storj.io/common v0.0.0-20221123115229-fed3e6651b63 // indirect
|
||||
storj.io/drpc v0.0.32 // indirect
|
||||
)
|
||||
|
|
|
|||
55
go.sum
55
go.sum
|
|
@ -1,38 +1,26 @@
|
|||
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
|
||||
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
|
||||
github.com/calebcase/tmpfile v1.0.3 h1:BZrOWZ79gJqQ3XbAQlihYZf/YCV0H4KPIdM5K5oMpJo=
|
||||
github.com/calebcase/tmpfile v1.0.3/go.mod h1:UAUc01aHeC+pudPagY/lWvt2qS9ZO5Zzof6/tIUzqeI=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/pprof v0.0.0-20211108044417-e9b028704de0 h1:rsq1yB2xiFLDYYaYdlGBsSkwVzsCo500wMhxvW5A/bk=
|
||||
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
||||
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
|
||||
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b h1:tO4MX3k5bvV0Sjv5jYrxStMTJxf1m/TW24XRyHji4aU=
|
||||
github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b/go.mod h1:q7yMR8BavTz/gBNtIT/uF487LMgcuEpNGKISLAjNQes=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0=
|
||||
github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
|
|
@ -46,41 +34,22 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
|
|||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
|
||||
github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
|
||||
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.19 h1:wqBb9bpD7jXkVi4XwIp8jn1fektaVBQ+cp9SHRXgAdo=
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.19/go.mod h1:kj1ViJhlyADa7DiA4xVnTuPA46lFKbM7mxQTrXCuJP4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.43.0 h1:Gy4sb32C98fbzVWZlTM1oTMdLWGyvxR03VhM6cBIU4g=
|
||||
github.com/valyala/fasthttp v1.43.0/go.mod h1:f6VbjjoI3z1NDOZOv17o6RvtRSWxC77seBFc2uWtgiY=
|
||||
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
||||
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 h1:zMsHhfK9+Wdl1F7sIKLyx3wrOFofpb3rWFbA4HgcK5k=
|
||||
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3/go.mod h1:R0Gbuw7ElaGSLOZUSwBm/GgVwMd30jWxBDdAyMOeTuc=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/assert v1.3.1 h1:vukIABvugfNMZMQO1ABsyQDJDTVQbn+LWSMy1ol1h6A=
|
||||
github.com/zeebo/assert v1.3.1/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg=
|
||||
github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ=
|
||||
github.com/zeebo/errs v1.3.0 h1:hmiaKqgYZzcVgRL1Vkc1Mn2914BbzB0IBxs+ebeutGs=
|
||||
github.com/zeebo/errs v1.3.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
|
||||
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
|
||||
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
|
||||
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
|
||||
|
|
@ -91,39 +60,26 @@ go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
|
|||
go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c=
|
||||
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220906165146-f3363e06e74c h1:yKufUcDwucU5urd+50/Opbt4AYpqthk7wHpHok8f1lo=
|
||||
golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde h1:ejfdSekXMDxDLbRrJMwUk6KnSLZ2McaUCVcIKM+N6jc=
|
||||
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
|
|
@ -139,24 +95,13 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
|||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI=
|
||||
gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
storj.io/common v0.0.0-20221123115229-fed3e6651b63 h1:OuleF/3FvZe3Nnu6NdwVr+FvCXjfD4iNNdgfI2kcs3k=
|
||||
storj.io/common v0.0.0-20221123115229-fed3e6651b63/go.mod h1:+gF7jbVvpjVIVHhK+EJFhfPbudX395lnPq/dKkj/Qys=
|
||||
storj.io/drpc v0.0.32 h1:5p5ZwsK/VOgapaCu+oxaPVwO6UwIs+iwdMiD50+R4PI=
|
||||
storj.io/drpc v0.0.32/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg=
|
||||
storj.io/uplink v1.10.0 h1:3hS0hszupHSxEoC4DsMpljaRy0uNoijEPVF6siIE28Q=
|
||||
storj.io/uplink v1.10.0/go.mod h1:gJIQumB8T3tBHPRive51AVpbc+v2xe+P/goFNMSRLG4=
|
||||
|
|
|
|||
114
main.go
114
main.go
|
|
@ -1,11 +1,9 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"strconv"
|
||||
|
||||
_ "github.com/joho/godotenv/autoload"
|
||||
"go.uber.org/zap"
|
||||
|
|
@ -13,62 +11,22 @@ import (
|
|||
)
|
||||
|
||||
var (
|
||||
httpAddr = ":9003"
|
||||
logLevel = "INFO"
|
||||
serverMode = "s3"
|
||||
httpAddr string
|
||||
logLevel string
|
||||
// obsSignedUrlExpiry time.Duration
|
||||
zapLogLevel zapcore.Level
|
||||
postFlagParse = []func(){}
|
||||
|
||||
registeredServers = []Server{
|
||||
&serverS3{},
|
||||
&serverStorj{},
|
||||
}
|
||||
mappedServers = map[string]Server{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
var err error
|
||||
_ = err
|
||||
|
||||
/* --- [preload] --- */
|
||||
var availableServerNames []string
|
||||
for _, s := range registeredServers {
|
||||
serverName := s.Name()
|
||||
if old, exist := mappedServers[serverName]; exist {
|
||||
panic(fmt.Sprintf("duplicate server name old: %T, new: %T", old, s))
|
||||
}
|
||||
mappedServers[serverName] = s
|
||||
availableServerNames = append(availableServerNames, serverName)
|
||||
}
|
||||
// app
|
||||
flag.StringVar(&httpAddr, "addr", os.Getenv("HTTP_ADDR"), "Server address")
|
||||
|
||||
/* --- app --- */
|
||||
var vHttpAddr = httpAddr
|
||||
if sHttpAddr := os.Getenv("HTTP_ADDR"); sHttpAddr != "" {
|
||||
vHttpAddr = sHttpAddr
|
||||
}
|
||||
flag.StringVar(&httpAddr, "addr", vHttpAddr, "Server address")
|
||||
|
||||
var vServerMode = serverMode
|
||||
if sServerMode := os.Getenv("SERVER_MODE"); sServerMode != "" {
|
||||
vServerMode = sServerMode
|
||||
}
|
||||
flag.StringVar(&serverMode, "server", vServerMode,
|
||||
fmt.Sprintf("Server mode (available [%s])", strings.Join(availableServerNames, ", ")))
|
||||
qpostFlagParse(func() {
|
||||
if httpAddr == "" {
|
||||
httpAddr = ":9003"
|
||||
}
|
||||
if serverMode == "" {
|
||||
serverMode = "s3"
|
||||
}
|
||||
})
|
||||
|
||||
/* --- log --- */
|
||||
var vLogLevel = logLevel
|
||||
if sLogLevel := os.Getenv("LOG_LEVEL"); sLogLevel != "" {
|
||||
vLogLevel = sLogLevel
|
||||
}
|
||||
flag.StringVar(&logLevel, "log-level", vLogLevel, "Log level")
|
||||
// log
|
||||
flag.StringVar(&logLevel, "log-level", os.Getenv("LOG_LEVEL"), "Log level")
|
||||
qpostFlagParse(func() {
|
||||
err := zapLogLevel.UnmarshalText([]byte(logLevel))
|
||||
if err != nil {
|
||||
|
|
@ -76,20 +34,20 @@ func init() {
|
|||
}
|
||||
})
|
||||
|
||||
/* --- OBS --- */
|
||||
if err = defaultObsOpts.Bind(flag.CommandLine); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// OBS
|
||||
flag.StringVar(&defaultObsOpts.Endpoint, "obs-endpoint", os.Getenv("OBS_ENDPOINT"), "OBS host")
|
||||
flag.StringVar(&defaultObsOpts.Region, "obs-region", os.Getenv("OBS_REGION"), "OBS region")
|
||||
flag.BoolVar(&defaultObsOpts.Secure, "obs-secure", ok1(strconv.ParseBool(os.Getenv("OBS_SECURE"))), "OBS secure transport")
|
||||
flag.StringVar(&defaultObsOpts.BucketName, "obs-bucket", os.Getenv("OBS_BUCKET_NAME"), "OBS bucket name")
|
||||
|
||||
/* --- OBS S3 --- */
|
||||
if err = defaultObsS3Opts.Bind(flag.CommandLine); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
flag.StringVar(&defaultObsOpts.HostRedirect, "obs-host-redirect", os.Getenv("OBS_HOST_REDIRECT"), "OBS host redirect")
|
||||
|
||||
/* --- OBS Storj (via LibUplink) --- */
|
||||
if err = defaultObsUplinkOpts.Bind(flag.CommandLine); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// obsSignedUrlExpiry, err = time.ParseDuration(os.Getenv("OBS_SIGNED_URL_EXPIRY"))
|
||||
// if err != nil {
|
||||
// // max signed value
|
||||
// obsSignedUrlExpiry = time.Duration(^uint64(0) / 2)
|
||||
// }
|
||||
// flag.DurationVar(&obsSignedUrlExpiry, "obs-signed-url-expiry", obsSignedUrlExpiry, "OBS ")
|
||||
}
|
||||
|
||||
func qpostFlagParse(f func()) {
|
||||
|
|
@ -114,35 +72,17 @@ func main() {
|
|||
sug := logger.Named("main").Sugar()
|
||||
sug.Infow("starting",
|
||||
"log_level", zapLogLevel,
|
||||
"server_mode", serverMode,
|
||||
// Generic OBS
|
||||
"obs_bucket", defaultObsOpts.BucketName,
|
||||
"obs_remove_bucket_name", defaultObsOpts.RemoveBucketName,
|
||||
"obs_redirect_secure", defaultObsOpts.RedirectSecure,
|
||||
"obs_endpoint", defaultObsOpts.Endpoint,
|
||||
"obs_host_redirect", defaultObsOpts.HostRedirect,
|
||||
"obs_redirect_code", defaultObsOpts.RedirectCode,
|
||||
"obs_url_expiry", defaultObsOpts.URLExpiry.String(),
|
||||
// S3
|
||||
"obs_s3_endpoint", defaultObsS3Opts.Endpoint,
|
||||
// Storj (via LibUplink)
|
||||
"obs_storj_satellite_addr", defaultObsUplinkOpts.SatelliteAddress,
|
||||
)
|
||||
|
||||
// lookup server mode handler
|
||||
srv, exist := mappedServers[serverMode]
|
||||
if !exist || srv == nil {
|
||||
sug.Fatalw("unknown server handler",
|
||||
"server_mode", serverMode)
|
||||
}
|
||||
|
||||
// run http server
|
||||
RunServer(context.Background(),
|
||||
srv,
|
||||
serverOptions{
|
||||
client := unwrap1(newObsClient(defaultObsOpts))
|
||||
srv.Init(serverOptions{
|
||||
Addr: httpAddr,
|
||||
Logger: logger.Named("server"),
|
||||
Opts: &defaultObsOpts,
|
||||
S3Opts: &defaultObsS3Opts,
|
||||
UplinkOpts: &defaultObsUplinkOpts,
|
||||
OBS: &defaultObsOpts,
|
||||
S3: client,
|
||||
})
|
||||
|
||||
srv.Run()
|
||||
}
|
||||
|
|
|
|||
129
obs.go
129
obs.go
|
|
@ -1,77 +1,102 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"unsafe"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
|
||||
_ "unsafe"
|
||||
)
|
||||
|
||||
type obsOptions struct {
|
||||
Endpoint string
|
||||
Region string
|
||||
Secure bool
|
||||
BucketName string
|
||||
RedirectSecure bool
|
||||
RedirectCode int // HTTP redirect status code
|
||||
URLExpiry time.Duration
|
||||
|
||||
HostRedirect string
|
||||
|
||||
RemoveBucketName bool
|
||||
}
|
||||
|
||||
var defaultObsOpts = obsOptions{
|
||||
URLExpiry: maxURLExpiry,
|
||||
RedirectCode: http.StatusMovedPermanently, // 301
|
||||
RemoveBucketName: false,
|
||||
var defaultObsOpts obsOptions
|
||||
|
||||
func newObsClient(opts obsOptions) (*minio.Client, error) {
|
||||
client, err := minio.New(opts.Endpoint, &minio.Options{
|
||||
Creds: credentials.NewEnvAWS(),
|
||||
BucketLookup: minio.BucketLookupAuto, // vhost / path
|
||||
Region: opts.Region,
|
||||
Secure: opts.Secure,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
setOverrideSignerType(client, credentials.SignatureV2)
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func (opts *obsOptions) Bind(fs *flag.FlagSet) (err error) {
|
||||
var (
|
||||
offsetCredsProvider uintptr
|
||||
offsetOverrideSignerType uintptr
|
||||
)
|
||||
|
||||
var vBucketName = opts.BucketName
|
||||
if sBucketName := os.Getenv("OBS_BUCKET_NAME"); sBucketName != "" {
|
||||
vBucketName = sBucketName
|
||||
func init() {
|
||||
vt := reflect.TypeOf(minio.Client{})
|
||||
if field, ok := vt.FieldByName("credsProvider"); ok {
|
||||
offsetCredsProvider = field.Offset
|
||||
} else {
|
||||
panic("cannot find credsProvider field")
|
||||
}
|
||||
fs.StringVar(&opts.BucketName, "obs-bucket", vBucketName, "OBS Bucket name")
|
||||
|
||||
var vRedirectSecure = opts.RedirectSecure
|
||||
if sRedirectSecure := os.Getenv("OBS_REDIRECT_SECURE"); sRedirectSecure != "" {
|
||||
vRedirectSecure, _ = strconv.ParseBool(sRedirectSecure)
|
||||
if field, ok := vt.FieldByName("overrideSignerType"); ok {
|
||||
offsetOverrideSignerType = field.Offset
|
||||
} else {
|
||||
panic("cannot find overrideSignerType field")
|
||||
}
|
||||
}
|
||||
fs.BoolVar(&opts.RedirectSecure, "obs-redirect-secure", vRedirectSecure, "OBS Redirect secure")
|
||||
|
||||
var vObsHostRedirect = opts.HostRedirect
|
||||
if sObsHostRedirect := os.Getenv("OBS_HOST_REDIRECT"); sObsHostRedirect != "" {
|
||||
vObsHostRedirect = sObsHostRedirect
|
||||
func getCredsProvider(client *minio.Client) *credentials.Credentials {
|
||||
return *(**credentials.Credentials)(unsafe.Add(unsafe.Pointer(client), offsetCredsProvider))
|
||||
}
|
||||
fs.StringVar(&opts.HostRedirect, "obs-host-redirect", vObsHostRedirect, "OBS Host redirect")
|
||||
|
||||
var vObsRedirectCode = opts.RedirectCode
|
||||
if sObsRedirectCode := os.Getenv("OBS_REDIRECT_CODE"); sObsRedirectCode != "" {
|
||||
var obsRedirectCode int64
|
||||
if obsRedirectCode, err = strconv.ParseInt(sObsRedirectCode, 10, 64); err != nil {
|
||||
err = errors.Wrap(err, "obs redirect code")
|
||||
return
|
||||
func setOverrideSignerType(client *minio.Client, signerType credentials.SignatureType) {
|
||||
ptr := (*credentials.SignatureType)(unsafe.Add(unsafe.Pointer(client), offsetOverrideSignerType))
|
||||
*ptr = signerType
|
||||
}
|
||||
vObsRedirectCode = int(obsRedirectCode)
|
||||
}
|
||||
fs.IntVar(&opts.RedirectCode, "obs-redirect-code", vObsRedirectCode, "OBS Redirect code")
|
||||
|
||||
var vObsUrlExpiry = opts.URLExpiry
|
||||
if sObsUrlExpiry := os.Getenv("OBS_URL_EXPIRY"); sObsUrlExpiry != "" {
|
||||
var obsUrlExpiry time.Duration
|
||||
if obsUrlExpiry, err = time.ParseDuration(sObsUrlExpiry); err != nil {
|
||||
err = errors.Wrap(err, "obs url expiry")
|
||||
return
|
||||
}
|
||||
vObsUrlExpiry = obsUrlExpiry
|
||||
}
|
||||
fs.DurationVar(&opts.URLExpiry, "obs-url-expiry", vObsUrlExpiry, "OBS Redirection URL expiry")
|
||||
//go:linkname isVirtualHostStyleRequest github.com/minio/minio-go/v7.(*Client).isVirtualHostStyleRequest
|
||||
func isVirtualHostStyleRequest(client *minio.Client, url url.URL, bucketName string) bool
|
||||
|
||||
var vObsRemoveBucketName = opts.RemoveBucketName
|
||||
if sObsRemoveBucketName := os.Getenv("OBS_REMOVE_BUCKET_NAME"); sObsRemoveBucketName != "" {
|
||||
vObsRemoveBucketName, _ = strconv.ParseBool(sObsRemoveBucketName)
|
||||
}
|
||||
fs.BoolVar(&opts.RemoveBucketName, "obs-remove-bucket-name", vObsRemoveBucketName, "OBS Remove Bucket name from prefix")
|
||||
return
|
||||
//go:linkname makeTargetURL github.com/minio/minio-go/v7.(*Client).makeTargetURL
|
||||
func makeTargetURL(client *minio.Client, bucketName, objectName, bucketLocation string, isVirtualHostStyle bool, queryValues url.Values) (*url.URL, error)
|
||||
|
||||
// requestMetadata - is container for all the values to make a request.
|
||||
type requestMetadata struct {
|
||||
// If set newRequest presigns the URL.
|
||||
presignURL bool
|
||||
|
||||
// User supplied.
|
||||
bucketName string
|
||||
objectName string
|
||||
queryValues url.Values
|
||||
customHeader http.Header
|
||||
extraPresignHeader http.Header
|
||||
expires int64
|
||||
|
||||
// Generated by our internal code.
|
||||
bucketLocation string
|
||||
contentBody io.Reader
|
||||
contentLength int64
|
||||
contentMD5Base64 string // carries base64 encoded md5sum
|
||||
contentSHA256Hex string // carries hex encoded sha256sum
|
||||
streamSha256 bool
|
||||
addCrc bool
|
||||
trailer http.Header // (http.Request).Trailer. Requires v4 signature.
|
||||
}
|
||||
|
||||
//go:linkname newRequest github.com/minio/minio-go/v7.(*Client).newRequest
|
||||
func newRequest(client *minio.Client, ctx context.Context, method string, metadata requestMetadata) (req *http.Request, err error)
|
||||
|
|
|
|||
124
obs_s3.go
124
obs_s3.go
|
|
@ -1,124 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
_ "unsafe"
|
||||
)
|
||||
|
||||
type obsS3Options struct {
|
||||
Endpoint string
|
||||
Region string
|
||||
Secure bool // S3 secure
|
||||
}
|
||||
|
||||
const maxURLExpiry = time.Duration(int64(^uint64(0) / 2))
|
||||
|
||||
var defaultObsS3Opts = obsS3Options{}
|
||||
|
||||
func (opts *obsS3Options) Bind(fs *flag.FlagSet) (err error) {
|
||||
fs.StringVar(&opts.Endpoint, "obs-endpoint", os.Getenv("OBS_ENDPOINT"), "OBS S3 Host")
|
||||
fs.StringVar(&opts.Region, "obs-region", os.Getenv("OBS_REGION"), "OBS S3 Region")
|
||||
|
||||
var vObsSecure = opts.Secure
|
||||
if sObsSecure := os.Getenv("OBS_SECURE"); sObsSecure != "" {
|
||||
var obsSecure bool
|
||||
if obsSecure, err = strconv.ParseBool(sObsSecure); err != nil {
|
||||
err = errors.Wrap(err, "obs secure")
|
||||
return
|
||||
}
|
||||
vObsSecure = obsSecure
|
||||
}
|
||||
fs.BoolVar(&opts.Secure, "obs-secure", vObsSecure, "OBS S3 secure transport")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func newObsS3Client(opts obsS3Options) (*minio.Client, error) {
|
||||
client, err := minio.New(opts.Endpoint, &minio.Options{
|
||||
Creds: credentials.NewEnvAWS(),
|
||||
BucketLookup: minio.BucketLookupAuto, // vhost / path
|
||||
Region: opts.Region,
|
||||
Secure: opts.Secure,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
setOverrideSignerType(client, credentials.SignatureV2)
|
||||
return client, nil
|
||||
}
|
||||
|
||||
var (
|
||||
offsetCredsProvider uintptr
|
||||
offsetOverrideSignerType uintptr
|
||||
)
|
||||
|
||||
func init() {
|
||||
vt := reflect.TypeOf(minio.Client{})
|
||||
if field, ok := vt.FieldByName("credsProvider"); ok {
|
||||
offsetCredsProvider = field.Offset
|
||||
} else {
|
||||
panic("cannot find credsProvider field")
|
||||
}
|
||||
|
||||
if field, ok := vt.FieldByName("overrideSignerType"); ok {
|
||||
offsetOverrideSignerType = field.Offset
|
||||
} else {
|
||||
panic("cannot find overrideSignerType field")
|
||||
}
|
||||
}
|
||||
|
||||
func getCredsProvider(client *minio.Client) *credentials.Credentials {
|
||||
return *(**credentials.Credentials)(unsafe.Add(unsafe.Pointer(client), offsetCredsProvider))
|
||||
}
|
||||
|
||||
func setOverrideSignerType(client *minio.Client, signerType credentials.SignatureType) {
|
||||
ptr := (*credentials.SignatureType)(unsafe.Add(unsafe.Pointer(client), offsetOverrideSignerType))
|
||||
*ptr = signerType
|
||||
}
|
||||
|
||||
//go:linkname isVirtualHostStyleRequest github.com/minio/minio-go/v7.(*Client).isVirtualHostStyleRequest
|
||||
func isVirtualHostStyleRequest(client *minio.Client, url url.URL, bucketName string) bool
|
||||
|
||||
//go:linkname makeTargetURL github.com/minio/minio-go/v7.(*Client).makeTargetURL
|
||||
func makeTargetURL(client *minio.Client, bucketName, objectName, bucketLocation string, isVirtualHostStyle bool, queryValues url.Values) (*url.URL, error)
|
||||
|
||||
// requestMetadata - is container for all the values to make a request.
|
||||
type requestMetadata struct {
|
||||
// If set newRequest presigns the URL.
|
||||
presignURL bool
|
||||
|
||||
// User supplied.
|
||||
bucketName string
|
||||
objectName string
|
||||
queryValues url.Values
|
||||
customHeader http.Header
|
||||
extraPresignHeader http.Header
|
||||
expires int64
|
||||
|
||||
// Generated by our internal code.
|
||||
bucketLocation string
|
||||
contentBody io.Reader
|
||||
contentLength int64
|
||||
contentMD5Base64 string // carries base64 encoded md5sum
|
||||
contentSHA256Hex string // carries hex encoded sha256sum
|
||||
streamSha256 bool
|
||||
addCrc bool
|
||||
trailer http.Header // (http.Request).Trailer. Requires v4 signature.
|
||||
}
|
||||
|
||||
//go:linkname newRequest github.com/minio/minio-go/v7.(*Client).newRequest
|
||||
func newRequest(client *minio.Client, ctx context.Context, method string, metadata requestMetadata) (req *http.Request, err error)
|
||||
179
obs_storj.go
179
obs_storj.go
|
|
@ -1,179 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"os"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"storj.io/uplink"
|
||||
"storj.io/uplink/edge"
|
||||
)
|
||||
|
||||
type obsStorjOptions struct {
|
||||
SatelliteAddress string
|
||||
APIKey string
|
||||
Passphrase string
|
||||
|
||||
AccessGrant string
|
||||
|
||||
AccessKeyID string
|
||||
ShareBaseURL string
|
||||
}
|
||||
|
||||
var defaultObsUplinkOpts = obsStorjOptions{
|
||||
// we can override satellite address from Access Grant with this
|
||||
// ex. "ap1.storj.io:7777"
|
||||
SatelliteAddress: "",
|
||||
ShareBaseURL: "https://link.storjshare.io",
|
||||
}
|
||||
|
||||
func (opts *obsStorjOptions) Bind(fs *flag.FlagSet) (err error) {
|
||||
var vSatelliteAddr = opts.SatelliteAddress
|
||||
if sSatelliteAddr := os.Getenv("UPLINK_SATELLITE_ADDR"); sSatelliteAddr != "" {
|
||||
vSatelliteAddr = sSatelliteAddr
|
||||
}
|
||||
fs.StringVar(&opts.SatelliteAddress, "uplink-satellite-addr", vSatelliteAddr, "OBS Storj Satellite Address")
|
||||
|
||||
{
|
||||
var vAPIKey = opts.APIKey
|
||||
if sAPIKey := os.Getenv("UPLINK_API_KEY"); sAPIKey != "" {
|
||||
vAPIKey = sAPIKey
|
||||
}
|
||||
fs.StringVar(&opts.APIKey, "uplink-api-key", vAPIKey, "OBS Storj API key")
|
||||
|
||||
var vPassphrase = opts.Passphrase
|
||||
if sPassphrase := os.Getenv("UPLINK_PASSPHRASE"); sPassphrase != "" {
|
||||
vPassphrase = sPassphrase
|
||||
}
|
||||
fs.StringVar(&opts.Passphrase, "uplink-passphrase", vPassphrase, "OBS Storj Passphrase")
|
||||
}
|
||||
|
||||
var vAccessGrant = opts.AccessGrant
|
||||
if sAccessGrant := os.Getenv("UPLINK_ACCESS_GRANT"); sAccessGrant != "" {
|
||||
vAccessGrant = sAccessGrant
|
||||
}
|
||||
fs.StringVar(&opts.AccessGrant, "uplink-access-grant", vAccessGrant, "OBS Storj Access Grant")
|
||||
|
||||
var vAccessKeyID = opts.AccessKeyID
|
||||
if sAccessKeyID := os.Getenv("UPLINK_ACCESS_KEY_ID"); sAccessKeyID != "" {
|
||||
vAccessKeyID = sAccessKeyID
|
||||
}
|
||||
fs.StringVar(&opts.AccessKeyID, "uplink-access-key-id", vAccessKeyID, "OBS Storj Access key ID")
|
||||
|
||||
var vShareBaseURL = opts.ShareBaseURL
|
||||
if sShareBaseURL := os.Getenv("UPLINK_SHARE_BASE_URL"); sShareBaseURL != "" {
|
||||
vShareBaseURL = sShareBaseURL
|
||||
}
|
||||
fs.StringVar(&opts.ShareBaseURL, "uplink-share-base-url", vShareBaseURL, "OBS Storj Link Share base URL")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
var defaultEdgeConfig = edge.Config{
|
||||
AuthServiceAddress: "auth.storjshare.io:7777",
|
||||
}
|
||||
|
||||
type storjAggegrateClient struct {
|
||||
edgeConfig *edge.Config
|
||||
access *uplink.Access
|
||||
project *uplink.Project
|
||||
|
||||
creds *edge.Credentials
|
||||
|
||||
accessKeyID string
|
||||
shareBaseURL string
|
||||
}
|
||||
|
||||
func (c *storjAggegrateClient) Init(ctx context.Context) (_ *storjAggegrateClient, err error) {
|
||||
if c.access != nil {
|
||||
// TODO: rate limit this, persist the registered access key ID
|
||||
if c.creds, err = c.edgeConfig.RegisterAccess(ctx, c.access, &edge.RegisterAccessOptions{
|
||||
// This will create `accessKeyID` that allow anonymous access
|
||||
// to any objects that `c.access` has access to (just like `public-read` ACL).
|
||||
Public: true,
|
||||
}); err != nil {
|
||||
err = errors.Wrap(err, "register access")
|
||||
return
|
||||
}
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (c *storjAggegrateClient) getAccessKeyID() (accessKeyID string) {
|
||||
accessKeyID = c.accessKeyID
|
||||
// fallback to `c.creds` if custom accessKeyID is not provided
|
||||
// and `c.creds` is specified.
|
||||
if accessKeyID == "" && c.creds != nil {
|
||||
accessKeyID = c.creds.AccessKeyID
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (c *storjAggegrateClient) getProject() *uplink.Project {
|
||||
return c.project
|
||||
}
|
||||
|
||||
func (c *storjAggegrateClient) JoinShareURL(bucket, key string, opts *edge.ShareURLOptions) (string, error) {
|
||||
accessKeyID := c.getAccessKeyID()
|
||||
return edge.JoinShareURL(c.shareBaseURL, accessKeyID, bucket, key, opts)
|
||||
}
|
||||
|
||||
func newObsStorjClient(ctx context.Context, opts obsStorjOptions) (client *storjAggegrateClient, err error) {
|
||||
var (
|
||||
access *uplink.Access
|
||||
project *uplink.Project
|
||||
)
|
||||
switch {
|
||||
case opts.AccessGrant != "":
|
||||
// Docs: https://docs.storj.io/dcs/concepts/access/access-grants
|
||||
access, err = uplink.ParseAccess(opts.AccessGrant)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "parse access")
|
||||
return
|
||||
}
|
||||
case opts.APIKey != "" && opts.Passphrase != "":
|
||||
// Docs: https://docs.storj.io/dcs/getting-started/quickstart-uplink-cli/generate-access-grants-and-tokens/generate-a-token
|
||||
if access, err = uplink.RequestAccessWithPassphrase(
|
||||
ctx,
|
||||
opts.SatelliteAddress,
|
||||
opts.APIKey,
|
||||
opts.Passphrase,
|
||||
); err != nil {
|
||||
err = errors.Wrap(err, "access with passphrase")
|
||||
return
|
||||
}
|
||||
default:
|
||||
// using custom accessKeyID
|
||||
access = nil
|
||||
}
|
||||
// TODO: limit access scope.
|
||||
if access != nil {
|
||||
// Consider to limit the access to specific bucket and prefix
|
||||
// access, err = access.Share(
|
||||
// uplink.ReadOnlyPermission(),
|
||||
// uplink.SharePrefix{
|
||||
// Bucket: "",
|
||||
// Prefix: "",
|
||||
// })
|
||||
|
||||
// open project
|
||||
project, err = uplink.OpenProject(ctx, access)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "open project")
|
||||
return
|
||||
}
|
||||
}
|
||||
// fallback default link sharing base url
|
||||
if opts.ShareBaseURL == "" {
|
||||
opts.ShareBaseURL = defaultObsUplinkOpts.ShareBaseURL
|
||||
}
|
||||
return (&storjAggegrateClient{
|
||||
edgeConfig: &defaultEdgeConfig,
|
||||
access: access,
|
||||
project: project,
|
||||
|
||||
accessKeyID: opts.AccessKeyID,
|
||||
shareBaseURL: opts.ShareBaseURL,
|
||||
}).Init(ctx)
|
||||
}
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"storj.io/uplink/edge"
|
||||
)
|
||||
|
||||
func TestStorjUplink(t *testing.T) {
|
||||
client, err := newObsStorjClient(context.Background(), obsStorjOptions{
|
||||
// Docs: https://docs.storj.io/dcs/getting-started/quickstart-uplink-cli/generate-access-grants-and-tokens/generate-a-token
|
||||
// SatelliteAddress: "link.storjshare.io",
|
||||
// APIKey: "xxx",
|
||||
// Passphrase: "xxxx",
|
||||
// --- or ---
|
||||
// Docs: https://docs.storj.io/dcs/concepts/access/access-grants
|
||||
// AccessGrant: "xxxx",
|
||||
// --- or ---
|
||||
// Docs: https://pkg.go.dev/storj.io/uplink/edge#Config.RegisterAccess
|
||||
// > RegisterAccess gets credentials for the Storj-hosted Gateway and linkshare service.
|
||||
// > All files accessible under the Access are then also accessible via those services.
|
||||
// > If you call this function a lot, and the use case allows it, please limit
|
||||
// > the lifetime of the credentials by setting Permission.NotAfter when creating the Access.
|
||||
AccessKeyID: "placeholder",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
if client.access != nil {
|
||||
println(client.access.SatelliteAddress())
|
||||
}
|
||||
|
||||
shareLinkURL, err := client.JoinShareURL(
|
||||
// "demo-bucket", "main.c",
|
||||
"moe", "moe-onl/13744453-430b-4e6b-ae81-29e7f2491317.png",
|
||||
&edge.ShareURLOptions{
|
||||
Raw: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
println(shareLinkURL)
|
||||
}
|
||||
148
server.go
148
server.go
|
|
@ -1,18 +1,19 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"bytes"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/s3utils"
|
||||
"github.com/minio/minio-go/v7/pkg/signer"
|
||||
"github.com/valyala/fasthttp"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrKind_ResourceNotFound = "OBS_RESOURCE_NOT_FOUND"
|
||||
ErrKind_MethodNotAllowed = "OBS_METHOD_NOT_ALLOWED"
|
||||
)
|
||||
|
||||
var (
|
||||
MethodGet = []byte(http.MethodGet)
|
||||
MethodHead = []byte(http.MethodHead)
|
||||
|
|
@ -21,37 +22,27 @@ var (
|
|||
type serverOptions struct {
|
||||
Addr string
|
||||
Logger *zap.Logger
|
||||
OBS *obsOptions
|
||||
|
||||
Opts *obsOptions
|
||||
S3Opts *obsS3Options
|
||||
UplinkOpts *obsStorjOptions
|
||||
ObjectExpiry time.Duration
|
||||
|
||||
S3 *minio.Client
|
||||
}
|
||||
|
||||
func (s *serverOptions) GetOpts() obsOptions {
|
||||
if s.Opts == nil {
|
||||
return defaultObsOpts
|
||||
}
|
||||
return *s.Opts
|
||||
type server struct {
|
||||
opts serverOptions
|
||||
logger *zap.SugaredLogger
|
||||
}
|
||||
|
||||
func (s *serverOptions) GetS3Opts() obsS3Options {
|
||||
if s.S3Opts == nil {
|
||||
return defaultObsS3Opts
|
||||
}
|
||||
return *s.S3Opts
|
||||
var srv server
|
||||
|
||||
func (s *server) Init(opts serverOptions) {
|
||||
s.opts = opts
|
||||
s.logger = opts.Logger.Sugar()
|
||||
}
|
||||
|
||||
func (s *serverOptions) GetUplinkOpts() obsStorjOptions {
|
||||
if s.UplinkOpts == nil {
|
||||
return defaultObsUplinkOpts
|
||||
}
|
||||
return *s.UplinkOpts
|
||||
}
|
||||
|
||||
func reportError(self interface {
|
||||
getLogger() *zap.SugaredLogger
|
||||
}, ctx *fasthttp.RequestCtx, errType string, err any) {
|
||||
self.getLogger().Errorw("handler error",
|
||||
func (s *server) reportError(ctx *fasthttp.RequestCtx, errType string, err any) {
|
||||
s.logger.Errorw("handler error",
|
||||
"kind", errType,
|
||||
"err", err)
|
||||
ctx.Response.Header.Set("x-error-code", errType)
|
||||
|
|
@ -67,20 +58,89 @@ func reportError(self interface {
|
|||
}
|
||||
}
|
||||
|
||||
type Server interface {
|
||||
Init(ctx context.Context, opts serverOptions) (err error)
|
||||
Name() string
|
||||
GetHandler() fasthttp.RequestHandler
|
||||
var (
|
||||
ErrKind_ResourceNotFound = "OBS_RESOURCE_NOT_FOUND"
|
||||
ErrKind_MethodNotAllowed = "OBS_METHOD_NOT_ALLOWED"
|
||||
ErrKind_S3ComposeRequest = "S3_COMPOSE_REQUEST"
|
||||
ErrKind_S3CredsProvider = "S3_CREDS_PROVIDER"
|
||||
)
|
||||
|
||||
func (s *server) handle(ctx *fasthttp.RequestCtx) {
|
||||
ctx.Response.Header.Set("server", "obs-access-signer")
|
||||
isMethodGet := bytes.Equal(ctx.Method(), MethodGet)
|
||||
isMethodHead := bytes.Equal(ctx.Method(), MethodHead)
|
||||
if !isMethodGet && !isMethodHead {
|
||||
ctx.SetStatusCode(http.StatusMethodNotAllowed)
|
||||
s.reportError(ctx, ErrKind_MethodNotAllowed, "")
|
||||
return
|
||||
}
|
||||
|
||||
func RunServer(ctx context.Context, s Server, opts serverOptions) {
|
||||
sug := opts.Logger.Sugar()
|
||||
s.Init(ctx, opts)
|
||||
sug.Infow("running server",
|
||||
"addr", opts.Addr)
|
||||
handler := s.GetHandler()
|
||||
fasthttp.ListenAndServe(opts.Addr, func(ctx *fasthttp.RequestCtx) {
|
||||
ctx.Response.Header.Set("server", "obs-access-signer")
|
||||
handler(ctx)
|
||||
})
|
||||
if isMethodHead {
|
||||
// Doc: https://www.rfc-editor.org/rfc/rfc9110.html#section-9.3.2-1
|
||||
ctx.Response.Header.Set("Content-Length", "0")
|
||||
}
|
||||
|
||||
bucketName := s.opts.OBS.BucketName
|
||||
isVirtualHostStyle := isVirtualHostStyleRequest(s.opts.S3, *s.opts.S3.EndpointURL(), bucketName)
|
||||
|
||||
path := ctx.Path()
|
||||
_path := bytes.TrimLeft(path, "/")
|
||||
objectName := unsafeByteSliceToString(_path)
|
||||
if _, _objectName, found := bytes.Cut(_path, []byte(bucketName)); !isVirtualHostStyle &&
|
||||
bytes.HasPrefix(_path, []byte(bucketName)) &&
|
||||
found {
|
||||
_objectName = bytes.TrimLeft(_objectName, "/")
|
||||
objectName = unsafeByteSliceToString(_objectName)
|
||||
}
|
||||
|
||||
// check if we had access to the object
|
||||
if _, err := s.opts.S3.StatObject(ctx, bucketName, objectName, minio.GetObjectOptions{}); err != nil {
|
||||
ctx.SetStatusCode(http.StatusNotFound)
|
||||
s.reportError(ctx, ErrKind_ResourceNotFound, err)
|
||||
return
|
||||
}
|
||||
|
||||
// compose initial request
|
||||
req, err := newRequest(s.opts.S3, ctx, http.MethodGet, requestMetadata{
|
||||
presignURL: true,
|
||||
bucketName: bucketName,
|
||||
objectName: objectName,
|
||||
expires: 1, // to trigger presigned generator
|
||||
queryValues: url.Values{},
|
||||
})
|
||||
if err != nil {
|
||||
ctx.SetStatusCode(http.StatusInternalServerError)
|
||||
s.reportError(ctx, ErrKind_S3ComposeRequest, err)
|
||||
return
|
||||
}
|
||||
|
||||
// grab creds from provider
|
||||
value, err := getCredsProvider(s.opts.S3).Get()
|
||||
if err != nil {
|
||||
ctx.SetStatusCode(http.StatusInternalServerError)
|
||||
s.reportError(ctx, ErrKind_S3CredsProvider, err)
|
||||
return
|
||||
}
|
||||
|
||||
// clear given params, set max signed value for expire, and re-presign.
|
||||
exp := strconv.FormatInt(int64(^uint64(0)/2), 10) // ~250years
|
||||
req.Header.Set("Expires", exp)
|
||||
req.URL.RawQuery = ""
|
||||
req = signer.PreSignV2(*req, value.AccessKeyID, value.SecretAccessKey, 0, isVirtualHostStyle)
|
||||
|
||||
// re-encode query string with Expires hack.
|
||||
query := req.URL.Query()
|
||||
query.Set("Expires", exp)
|
||||
req.URL.RawQuery = s3utils.QueryEncode(query)
|
||||
if hostRedirect := s.opts.OBS.HostRedirect; hostRedirect != "" {
|
||||
req.URL.Host = hostRedirect
|
||||
}
|
||||
|
||||
ctx.Redirect(req.URL.String(), http.StatusMovedPermanently)
|
||||
}
|
||||
|
||||
func (s *server) Run() {
|
||||
s.logger.Infow("running server",
|
||||
"addr", s.opts.Addr)
|
||||
fasthttp.ListenAndServe(s.opts.Addr, s.handle)
|
||||
}
|
||||
|
|
|
|||
166
server_s3.go
166
server_s3.go
|
|
@ -1,166 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/s3utils"
|
||||
"github.com/minio/minio-go/v7/pkg/signer"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/valyala/fasthttp"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type serverS3 struct {
|
||||
opts obsOptions
|
||||
s3opts obsS3Options
|
||||
|
||||
logger *zap.SugaredLogger
|
||||
|
||||
s3c *minio.Client
|
||||
}
|
||||
|
||||
func (s *serverS3) Init(ctx context.Context, opts serverOptions) (err error) {
|
||||
s.opts = opts.GetOpts()
|
||||
s.s3opts = opts.GetS3Opts()
|
||||
|
||||
s.logger = opts.Logger.Named(s.Name()).Sugar()
|
||||
|
||||
if s.s3c, err = newObsS3Client(s.s3opts); err != nil {
|
||||
err = errors.Wrap(err, "obs s3 client")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (s *serverS3) Name() string {
|
||||
return "s3"
|
||||
}
|
||||
|
||||
func (s *serverS3) getLogger() *zap.SugaredLogger { return s.logger }
|
||||
func (s *serverS3) reportError(ctx *fasthttp.RequestCtx, errType string, err any) {
|
||||
reportError(s, ctx, errType, err)
|
||||
}
|
||||
|
||||
var (
|
||||
ErrKind_S3ComposeRequest = "S3_COMPOSE_REQUEST"
|
||||
ErrKind_S3CredsProvider = "S3_CREDS_PROVIDER"
|
||||
)
|
||||
|
||||
func (s *serverS3) handle(ctx *fasthttp.RequestCtx) {
|
||||
isMethodGet := bytes.Equal(ctx.Method(), MethodGet)
|
||||
isMethodHead := bytes.Equal(ctx.Method(), MethodHead)
|
||||
if !isMethodGet && !isMethodHead {
|
||||
ctx.SetStatusCode(http.StatusMethodNotAllowed)
|
||||
s.reportError(ctx, ErrKind_MethodNotAllowed, "")
|
||||
return
|
||||
}
|
||||
|
||||
if isMethodHead {
|
||||
// Doc: https://www.rfc-editor.org/rfc/rfc9110.html#section-9.3.2-1
|
||||
ctx.Response.Header.Set("Content-Length", "0")
|
||||
}
|
||||
|
||||
bucketName := s.opts.BucketName
|
||||
isVirtualHostStyle := isVirtualHostStyleRequest(s.s3c, *s.s3c.EndpointURL(), bucketName)
|
||||
|
||||
path := ctx.Path()
|
||||
_path := bytes.TrimLeft(path, "/")
|
||||
if s.opts.RemoveBucketName {
|
||||
if _, _pathWithoutBucketName, found := bytes.Cut(_path, []byte(`/`)); found {
|
||||
// no need to check `isVirtualHostStyle` since this is our own implementation of handling request URI
|
||||
_path = _pathWithoutBucketName
|
||||
}
|
||||
}
|
||||
objectName := unsafeByteSliceToString(_path)
|
||||
|
||||
s.logger.Debugw("handle",
|
||||
"bucket", bucketName,
|
||||
"objectName", objectName)
|
||||
|
||||
// check if we had access to the object
|
||||
if meta, err := s.s3c.StatObject(ctx, bucketName, objectName, minio.GetObjectOptions{}); err != nil {
|
||||
ctx.SetStatusCode(http.StatusNotFound)
|
||||
s.reportError(ctx, ErrKind_ResourceNotFound, err)
|
||||
return
|
||||
} else {
|
||||
_ = meta
|
||||
}
|
||||
|
||||
// compose initial request
|
||||
expireSeconds := int64(s.opts.URLExpiry / time.Second)
|
||||
req, err := newRequest(s.s3c, ctx, http.MethodGet, requestMetadata{
|
||||
presignURL: true,
|
||||
bucketName: bucketName,
|
||||
objectName: objectName,
|
||||
expires: expireSeconds, // to trigger presigned generator
|
||||
queryValues: url.Values{},
|
||||
})
|
||||
if err != nil {
|
||||
ctx.SetStatusCode(http.StatusInternalServerError)
|
||||
s.reportError(ctx, ErrKind_S3ComposeRequest, err)
|
||||
return
|
||||
}
|
||||
|
||||
// grab creds from provider
|
||||
value, err := getCredsProvider(s.s3c).Get()
|
||||
if err != nil {
|
||||
ctx.SetStatusCode(http.StatusInternalServerError)
|
||||
s.reportError(ctx, ErrKind_S3CredsProvider, err)
|
||||
return
|
||||
}
|
||||
|
||||
var statusCode = s.opts.RedirectCode
|
||||
|
||||
// custom "expiry"
|
||||
var exp string
|
||||
if expiry := s.opts.URLExpiry; expiry == maxURLExpiry || expiry <= 0 {
|
||||
// clear given params, set max signed value for expire, and re-presign.
|
||||
exp = strconv.FormatInt(int64(^uint64(0)/2), 10) // ~250years
|
||||
} else {
|
||||
// we can't allow a permanent redirect here since we already have
|
||||
// expiry set, the redirected url needs to be updated.
|
||||
if statusCode == http.StatusMovedPermanently || (statusCode < 300 || statusCode > 399) {
|
||||
statusCode = http.StatusTemporaryRedirect
|
||||
}
|
||||
|
||||
expireAt := time.Now().UTC().Add(s.opts.URLExpiry)
|
||||
exp = strconv.FormatInt(int64(expireAt.Unix()), 10)
|
||||
// set redirect cache lifetime
|
||||
if statusCode == http.StatusTemporaryRedirect {
|
||||
ctx.Response.Header.Set("Cache-Control", fmt.Sprintf("max-age=%d", expireSeconds))
|
||||
ctx.Response.Header.Set("Expires", expireAt.Format("Mon, 02 Jan 2006 15:04:05 GMT"))
|
||||
}
|
||||
}
|
||||
req.Header.Set("Expires", exp)
|
||||
req.URL.RawQuery = ""
|
||||
req = signer.PreSignV2(*req, value.AccessKeyID, value.SecretAccessKey, 0, isVirtualHostStyle)
|
||||
|
||||
// re-encode query string with Expires hack.
|
||||
query := req.URL.Query()
|
||||
query.Set("Expires", exp)
|
||||
req.URL.RawQuery = s3utils.QueryEncode(query)
|
||||
|
||||
if s.opts.RedirectSecure {
|
||||
req.URL.Scheme = "https"
|
||||
} else {
|
||||
req.URL.Scheme = "http"
|
||||
}
|
||||
|
||||
if hostRedirect := s.opts.HostRedirect; hostRedirect != "" {
|
||||
req.URL.Host = hostRedirect
|
||||
}
|
||||
|
||||
ctx.Redirect(req.URL.String(), statusCode)
|
||||
}
|
||||
|
||||
func (s *serverS3) GetHandler() fasthttp.RequestHandler {
|
||||
return s.handle
|
||||
}
|
||||
117
server_storj.go
117
server_storj.go
|
|
@ -1,117 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/valyala/fasthttp"
|
||||
"go.uber.org/zap"
|
||||
"storj.io/uplink/edge"
|
||||
)
|
||||
|
||||
type serverStorj struct {
|
||||
opts obsOptions
|
||||
logger *zap.SugaredLogger
|
||||
|
||||
sc *storjAggegrateClient
|
||||
}
|
||||
|
||||
func (s *serverStorj) Init(ctx context.Context, opts serverOptions) (err error) {
|
||||
s.opts = opts.GetOpts()
|
||||
s.logger = opts.Logger.Named(s.Name()).Sugar()
|
||||
{
|
||||
if s.sc, err = newObsStorjClient(ctx, opts.GetUplinkOpts()); err != nil {
|
||||
err = errors.Wrap(err, "obs uplink client")
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *serverStorj) Name() string {
|
||||
return "storj"
|
||||
}
|
||||
|
||||
func (s *serverStorj) getLogger() *zap.SugaredLogger { return s.logger }
|
||||
func (s *serverStorj) reportError(ctx *fasthttp.RequestCtx, errType string, err any) {
|
||||
reportError(s, ctx, errType, err)
|
||||
}
|
||||
|
||||
var (
|
||||
ErrKind_StorjComposeShareURL = "STORJ_COMPOSE_SHARE_URL"
|
||||
)
|
||||
|
||||
func (s *serverStorj) handle(ctx *fasthttp.RequestCtx) {
|
||||
isMethodGet := bytes.Equal(ctx.Method(), MethodGet)
|
||||
isMethodHead := bytes.Equal(ctx.Method(), MethodHead)
|
||||
if !(isMethodGet || isMethodHead) {
|
||||
ctx.SetStatusCode(http.StatusMethodNotAllowed)
|
||||
s.reportError(ctx, ErrKind_MethodNotAllowed, "")
|
||||
return
|
||||
}
|
||||
|
||||
if isMethodHead {
|
||||
// Doc: https://www.rfc-editor.org/rfc/rfc9110.html#section-9.3.2-1
|
||||
ctx.Response.Header.Set("Content-Length", "0")
|
||||
}
|
||||
|
||||
bucketName := s.opts.BucketName
|
||||
path := ctx.Path()
|
||||
_path := bytes.TrimLeft(path, "/")
|
||||
if s.opts.RemoveBucketName {
|
||||
if _, _pathWithoutBucketName, found := bytes.Cut(_path, []byte(`/`)); found {
|
||||
// no need to check `isVirtualHostStyle` since this is our own implementation of handling request URI
|
||||
_path = _pathWithoutBucketName
|
||||
}
|
||||
}
|
||||
objectName := unsafeByteSliceToString(_path)
|
||||
|
||||
s.logger.Debugw("handle",
|
||||
"bucket", bucketName,
|
||||
"objectName", objectName)
|
||||
|
||||
// use project
|
||||
if project := s.sc.getProject(); project != nil {
|
||||
// check if we had access to the object
|
||||
if meta, err := project.StatObject(ctx, bucketName, objectName); err != nil {
|
||||
ctx.SetStatusCode(http.StatusNotFound)
|
||||
s.reportError(ctx, ErrKind_ResourceNotFound, err)
|
||||
return
|
||||
} else {
|
||||
_ = meta
|
||||
}
|
||||
}
|
||||
|
||||
shareURL, err := s.sc.JoinShareURL(bucketName, objectName, &edge.ShareURLOptions{
|
||||
Raw: true,
|
||||
})
|
||||
if err != nil {
|
||||
ctx.SetStatusCode(http.StatusInternalServerError)
|
||||
s.reportError(ctx, ErrKind_StorjComposeShareURL, err)
|
||||
}
|
||||
|
||||
var statusCode = s.opts.RedirectCode
|
||||
|
||||
if statusCode < 300 || statusCode >= 399 {
|
||||
// fallback of invalid redirect code
|
||||
statusCode = http.StatusTemporaryRedirect
|
||||
}
|
||||
|
||||
expireAt := time.Now().UTC().Add(s.opts.URLExpiry)
|
||||
expireSeconds := int64(s.opts.URLExpiry / time.Second)
|
||||
// set redirect cache lifetime
|
||||
if statusCode == http.StatusTemporaryRedirect {
|
||||
ctx.Response.Header.Set("Cache-Control", fmt.Sprintf("max-age=%d", expireSeconds))
|
||||
ctx.Response.Header.Set("Expires", expireAt.Format("Mon, 02 Jan 2006 15:04:05 GMT"))
|
||||
}
|
||||
|
||||
ctx.Redirect(shareURL, s.opts.RedirectCode)
|
||||
}
|
||||
|
||||
func (s *serverStorj) GetHandler() fasthttp.RequestHandler {
|
||||
return s.handle
|
||||
}
|
||||
Loading…
Reference in a new issue