package s3store import ( "bytes" "context" "crypto/rand" "encoding/hex" "errors" "fmt" "io" "os" "path" "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/smithy-go" "github.com/npfaro/dumbforge/internal/remoteurl" ) type Object struct { Data []byte ETag string } type Store struct { client *s3.Client spec remoteurl.Spec } func New(ctx context.Context, spec remoteurl.Spec) (*Store, error) { opts := []func(*config.LoadOptions) error{config.WithRegion("auto")} if spec.Profile != "" { opts = append(opts, config.WithSharedConfigProfile(spec.Profile)) } cfg, err := config.LoadDefaultConfig(ctx, opts...) if err != nil { return nil, fmt.Errorf("load AWS configuration: %w", err) } client := s3.NewFromConfig(cfg, func(o *s3.Options) { o.BaseEndpoint = aws.String(spec.Endpoint) o.UsePathStyle = true // S3-compatible services do not uniformly support the SDK's optional // streaming CRC32 trailer, especially when Content-Encoding is set. o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired }) return &Store{client: client, spec: spec}, nil } func (s *Store) Spec() remoteurl.Spec { return s.spec } func (s *Store) key(name string) string { return path.Join(strings.Trim(s.spec.Prefix, "/"), strings.TrimLeft(name, "/")) } // RootKey returns the object key used for the repository's HTML landing page. // It intentionally has no trailing slash; Git can still append /info/refs to // the same URL when cloning. func (s *Store) RootKey() string { return strings.Trim(s.spec.Prefix, "/") } func (s *Store) Get(ctx context.Context, name string) (Object, error) { out, err := s.client.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(s.spec.Bucket), Key: aws.String(s.key(name)), }) if err != nil { return Object{}, err } defer out.Body.Close() data, err := io.ReadAll(out.Body) if err != nil { return Object{}, fmt.Errorf("read s3://%s/%s: %w", s.spec.Bucket, s.key(name), err) } return Object{Data: data, ETag: aws.ToString(out.ETag)}, nil } func (s *Store) Put(ctx context.Context, name string, data []byte, contentType, cacheControl string) error { return s.putBytes(ctx, s.key(name), data, contentType, cacheControl, "") } func (s *Store) PutEncoded(ctx context.Context, name string, data []byte, contentType, cacheControl, contentEncoding string) error { return s.putBytes(ctx, s.key(name), data, contentType, cacheControl, contentEncoding) } func (s *Store) PutRoot(ctx context.Context, data []byte, contentType, cacheControl string) error { return s.putBytes(ctx, s.RootKey(), data, contentType, cacheControl, "") } func (s *Store) PutRootEncoded(ctx context.Context, data []byte, contentType, cacheControl, contentEncoding string) error { return s.putBytes(ctx, s.RootKey(), data, contentType, cacheControl, contentEncoding) } func (s *Store) putBytes(ctx context.Context, key string, data []byte, contentType, cacheControl, contentEncoding string) error { input := &s3.PutObjectInput{ Bucket: aws.String(s.spec.Bucket), Key: aws.String(key), Body: bytes.NewReader(data), ContentType: aws.String(contentType), } if cacheControl != "" { input.CacheControl = aws.String(cacheControl) } if contentEncoding != "" { input.ContentEncoding = aws.String(contentEncoding) } _, err := s.client.PutObject(ctx, input) if err != nil { return fmt.Errorf("put s3://%s/%s: %w", s.spec.Bucket, key, err) } return nil } func (s *Store) PutFile(ctx context.Context, name, filename, contentType, cacheControl string) error { // #nosec G304 -- callers pass paths produced in our private pack directory. f, err := os.Open(filename) if err != nil { return err } defer f.Close() input := &s3.PutObjectInput{ Bucket: aws.String(s.spec.Bucket), Key: aws.String(s.key(name)), Body: f, ContentType: aws.String(contentType), } if cacheControl != "" { input.CacheControl = aws.String(cacheControl) } _, err = s.client.PutObject(ctx, input) if err != nil { return fmt.Errorf("put s3://%s/%s: %w", s.spec.Bucket, s.key(name), err) } return nil } func (s *Store) Delete(ctx context.Context, name string) error { _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: aws.String(s.spec.Bucket), Key: aws.String(s.key(name)), }) if err != nil { return fmt.Errorf("delete s3://%s/%s: %w", s.spec.Bucket, s.key(name), err) } return nil } type Lock struct { store *Store name string token string etag string refreshedAt time.Time } func (s *Store) AcquireLock(ctx context.Context) (*Lock, error) { random := make([]byte, 16) if _, err := rand.Read(random); err != nil { return nil, err } token := hex.EncodeToString(random) body := []byte(fmt.Sprintf("%s\n%s\n", token, time.Now().UTC().Format(time.RFC3339Nano))) name := ".dumbforge/lock" input := &s3.PutObjectInput{ Bucket: aws.String(s.spec.Bucket), Key: aws.String(s.key(name)), Body: bytes.NewReader(body), ContentType: aws.String("text/plain; charset=utf-8"), IfNoneMatch: aws.String("*"), } out, err := s.client.PutObject(ctx, input) if err != nil { if IsPreconditionFailed(err) { existing, getErr := s.Get(ctx, name) if getErr != nil { return nil, fmt.Errorf("repository is locked by another push") } lines := strings.Split(string(existing.Data), "\n") if len(lines) < 2 { return nil, fmt.Errorf("repository is locked by another push") } lockedAt, timeErr := time.Parse(time.RFC3339Nano, lines[1]) if timeErr != nil || time.Since(lockedAt) < time.Hour { return nil, fmt.Errorf("repository is locked by another push") } input.IfNoneMatch = nil input.IfMatch = aws.String(existing.ETag) out, err = s.client.PutObject(ctx, input) if err != nil { return nil, fmt.Errorf("replace stale repository lock: %w", err) } } else { return nil, fmt.Errorf("acquire repository lock: %w", err) } } return &Lock{store: s, name: name, token: token, etag: aws.ToString(out.ETag), refreshedAt: time.Now()}, nil } // Refresh proves that this process still owns the lease before it publishes // mutable repository metadata. func (l *Lock) Refresh(ctx context.Context) error { body := []byte(fmt.Sprintf("%s\n%s\n", l.token, time.Now().UTC().Format(time.RFC3339Nano))) out, err := l.store.client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(l.store.spec.Bucket), Key: aws.String(l.store.key(l.name)), Body: bytes.NewReader(body), ContentType: aws.String("text/plain; charset=utf-8"), IfMatch: aws.String(l.etag), }) if err != nil { return fmt.Errorf("refresh repository lock: %w", err) } l.etag = aws.ToString(out.ETag) l.refreshedAt = time.Now() return nil } // RefreshIfOlderThan avoids an S3 round trip for ordinary pushes while still // renewing the lease during unusually long publications. func (l *Lock) RefreshIfOlderThan(ctx context.Context, age time.Duration) error { if time.Since(l.refreshedAt) < age { return nil } return l.Refresh(ctx) } func (l *Lock) Release(ctx context.Context) error { _, err := l.store.client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: aws.String(l.store.spec.Bucket), Key: aws.String(l.store.key(l.name)), IfMatch: aws.String(l.etag), }) if err != nil { return fmt.Errorf("release repository lock: %w", err) } return nil } func IsNotFound(err error) bool { var apiErr smithy.APIError if !errors.As(err, &apiErr) { return false } switch apiErr.ErrorCode() { case "NoSuchKey", "NotFound", "404": return true default: return false } } func IsPreconditionFailed(err error) bool { var apiErr smithy.APIError if !errors.As(err, &apiErr) { return false } return apiErr.ErrorCode() == "PreconditionFailed" || apiErr.ErrorCode() == "412" }