package publish import ( "context" "fmt" "path" "sort" "strings" "sync" "time" "github.com/npfaro/dumbforge/internal/gitutil" "github.com/npfaro/dumbforge/internal/s3store" ) const ( mutableCache = "no-cache, max-age=0, must-revalidate" immutableCache = "public, max-age=31536000, immutable" smallPushMaxBytes = 8 << 20 smallPushMaxObjects = 64 ) type Update struct { Src string Dst string Force bool } type AppliedUpdate struct { Dst string OldOID string NewOID string } type Result struct { Updates []AppliedUpdate PackName string PackObjects uint32 Head string Branches []string SiteVersion string } type Publisher struct { Store *s3store.Store Git gitutil.Git Progress func(format string, args ...any) AfterPublish func(context.Context, Result) prepared *preparedPush } type preparedPush struct { lock *s3store.Lock metadata Metadata } func (p *Publisher) List(ctx context.Context) (Metadata, error) { return LoadMetadata(ctx, p.Store) } // Prepare serializes a push before refs are advertised. Holding the lock from // advertisement through publication removes a redundant metadata read without // weakening the concurrent-update check. func (p *Publisher) Prepare(ctx context.Context) (Metadata, error) { if p.prepared != nil { return p.prepared.metadata, nil } lock, err := p.Store.AcquireLock(ctx) if err != nil { return Metadata{}, err } metadata, err := LoadMetadata(ctx, p.Store) if err != nil { _ = lock.Release(context.WithoutCancel(ctx)) return Metadata{}, err } p.prepared = &preparedPush{lock: lock, metadata: metadata} return metadata, nil } // Abort releases a prepared push if the remote-helper session ends before it // sends a push command. func (p *Publisher) Abort(ctx context.Context) error { if p.prepared == nil { return nil } lock := p.prepared.lock p.prepared = nil return lock.Release(ctx) } // Push applies a batch as one publication. expected contains the refs advertised // to Git before it sent the push commands and prevents a concurrent update from // being silently overwritten. func (p *Publisher) Push(ctx context.Context, updates []Update, expected map[string]string) (result Result, err error) { if len(updates) == 0 { return Result{}, nil } shallow, err := p.Git.IsShallow(ctx) if err != nil { return Result{}, err } if shallow { return Result{}, fmt.Errorf("cannot publish from a shallow repository; fetch the complete history first") } var lock *s3store.Lock var metadata Metadata if p.prepared != nil { lock = p.prepared.lock metadata = p.prepared.metadata p.prepared = nil } else { lock, err = p.Store.AcquireLock(ctx) if err != nil { return Result{}, err } metadata, err = LoadMetadata(ctx, p.Store) if err != nil { _ = lock.Release(context.WithoutCancel(ctx)) return Result{}, err } } defer func() { if releaseErr := lock.Release(context.WithoutCancel(ctx)); err == nil && releaseErr != nil { err = releaseErr } }() if err := checkExpected(metadata, updates, expected); err != nil { return Result{}, err } resolved, err := p.resolveAndValidate(ctx, metadata, updates) if err != nil { return Result{}, err } wants := make([]string, 0, len(resolved)) for _, update := range resolved { if update.NewOID != "" { wants = append(wants, update.NewOID) } } haves := make([]string, 0, len(metadata.Refs)) for _, ref := range metadata.Refs { haves = append(haves, ref.OID) } pack, err := p.Git.PackObjects(ctx, wants, haves) if err != nil { return Result{}, err } defer pack.Close() packAdded := false refsPublishedEarly := false if pack != nil && pack.Objects > 0 { looseObjects, useLoose, looseErr := p.Git.SmallLooseObjects(ctx, pack, smallPushMaxObjects, smallPushMaxBytes) if looseErr != nil { return Result{}, looseErr } if useLoose { p.progress("uploading %d loose objects", len(looseObjects)) publications := make([]func() error, 0, len(looseObjects)+len(resolved)) for _, object := range looseObjects { object := object publications = append(publications, func() error { return p.Store.PutFile(ctx, path.Join("objects", object.OID[:2], object.OID[2:]), object.Path, "application/x-git-loose-object", immutableCache) }) } // Individual ref files are not used for discovery by dumb HTTP; publish // them in the same wave and update the authoritative info/refs only // after every object and ref write has succeeded. for _, update := range resolved { update := update if update.OldOID == update.NewOID { continue } if update.NewOID == "" { publications = append(publications, func() error { return p.Store.Delete(ctx, update.Dst) }) } else { publications = append(publications, func() error { return p.Store.Put(ctx, update.Dst, []byte(update.NewOID+"\n"), "text/plain; charset=utf-8", mutableCache) }) } } if err := runParallel(publications...); err != nil { return Result{}, err } refsPublishedEarly = true result.PackObjects = pack.Objects } else { packName := "pack-" + pack.Hash + ".pack" indexName := "pack-" + pack.Hash + ".idx" p.progress("uploading %s (%d objects)", packName, pack.Objects) if err := runParallel( func() error { return p.Store.PutFile(ctx, path.Join("objects/pack", packName), pack.PackPath, "application/x-git-packed-objects", immutableCache) }, func() error { return p.Store.PutFile(ctx, path.Join("objects/pack", indexName), pack.IndexPath, "application/x-git-packed-objects-toc", immutableCache) }, ); err != nil { return Result{}, err } metadata.Packs[packName] = struct{}{} result.PackName = packName result.PackObjects = pack.Objects packAdded = true } } previousHead := metadata.Head for _, update := range resolved { result.Updates = append(result.Updates, update) if update.NewOID == "" { delete(metadata.Refs, update.Dst) continue } ref := Ref{OID: update.NewOID} if strings.HasPrefix(update.Dst, "refs/tags/") { peeled, peelErr := p.Git.Peel(ctx, update.NewOID) if peelErr == nil && peeled != update.NewOID { ref.Peeled = peeled } } metadata.Refs[update.Dst] = ref } if metadata.Head == "" { metadata.Head = p.chooseInitialHead(ctx, resolved) } result.Head = metadata.Head result.Branches = metadata.BranchNames() result.SiteVersion = metadata.SiteVersion if err := lock.RefreshIfOlderThan(ctx, 30*time.Minute); err != nil { return Result{}, err } // Packs are already durable. Publish the remaining internal pointers in // parallel, then info/refs last so dumb-HTTP clients cannot discover a ref // before all of the objects needed to read it are available. var publications []func() error if packAdded { publications = append(publications, func() error { return p.Store.Put(ctx, "objects/info/packs", renderPackList(metadata.Packs), "text/plain; charset=utf-8", mutableCache) }) } if !refsPublishedEarly { for _, update := range resolved { update := update if update.OldOID == update.NewOID { continue } if update.NewOID == "" { publications = append(publications, func() error { return p.Store.Delete(ctx, update.Dst) }) continue } publications = append(publications, func() error { return p.Store.Put(ctx, update.Dst, []byte(update.NewOID+"\n"), "text/plain; charset=utf-8", mutableCache) }) } } if metadata.Head != "" && metadata.Head != previousHead { publications = append(publications, func() error { return p.Store.Put(ctx, "HEAD", []byte("ref: "+metadata.Head+"\n"), "text/plain; charset=utf-8", mutableCache) }) } if err := runParallel(publications...); err != nil { return Result{}, err } if err := p.Store.Put(ctx, "info/refs", renderInfoRefs(metadata.Refs), "text/plain; charset=utf-8", mutableCache); err != nil { return Result{}, err } p.progress("published %d ref update(s)", len(resolved)) if p.AfterPublish != nil { p.AfterPublish(ctx, result) } return result, nil } func (p *Publisher) resolveAndValidate(ctx context.Context, metadata Metadata, updates []Update) ([]AppliedUpdate, error) { seen := map[string]struct{}{} resolved := make([]AppliedUpdate, 0, len(updates)) for _, update := range updates { if err := p.Git.CheckRef(ctx, update.Dst); err != nil { return nil, err } if _, ok := seen[update.Dst]; ok { return nil, fmt.Errorf("destination ref %s appears more than once", update.Dst) } seen[update.Dst] = struct{}{} oldOID := metadata.Refs[update.Dst].OID if update.Src == "" { if update.Dst == metadata.Head { return nil, fmt.Errorf("refusing to delete the remote HEAD branch %s", update.Dst) } resolved = append(resolved, AppliedUpdate{Dst: update.Dst, OldOID: oldOID}) continue } newOID, err := p.Git.RevParse(ctx, update.Src) if err != nil { return nil, fmt.Errorf("resolve %s: %w", update.Src, err) } if oldOID != "" && oldOID != newOID && !update.Force { if !strings.HasPrefix(update.Dst, "refs/heads/") { return nil, fmt.Errorf("ref %s already exists; use force to replace it", update.Dst) } if !p.Git.ObjectExists(ctx, oldOID) { return nil, fmt.Errorf("remote tip %s is missing locally; fetch before pushing", oldOID) } ancestor, err := p.Git.IsAncestor(ctx, oldOID, newOID) if err != nil { return nil, err } if !ancestor { return nil, fmt.Errorf("non-fast-forward update to %s", update.Dst) } } resolved = append(resolved, AppliedUpdate{Dst: update.Dst, OldOID: oldOID, NewOID: newOID}) } return resolved, nil } func checkExpected(metadata Metadata, updates []Update, expected map[string]string) error { if expected == nil { return nil } for _, update := range updates { current := metadata.Refs[update.Dst].OID if advertised, ok := expected[update.Dst]; ok && advertised != current { return fmt.Errorf("remote ref %s changed during push; retry", update.Dst) } if _, ok := expected[update.Dst]; !ok && current != "" { return fmt.Errorf("remote ref %s was created during push; retry", update.Dst) } } return nil } func chooseHead(updates []AppliedUpdate) string { for _, preferred := range []string{"refs/heads/main", "refs/heads/master"} { for _, update := range updates { if update.Dst == preferred && update.NewOID != "" { return preferred } } } names := make([]string, 0, len(updates)) for _, update := range updates { if strings.HasPrefix(update.Dst, "refs/heads/") && update.NewOID != "" { names = append(names, update.Dst) } } sort.Strings(names) if len(names) > 0 { return names[0] } return "" } func (p *Publisher) chooseInitialHead(ctx context.Context, updates []AppliedUpdate) string { if localHead, err := p.Git.SymbolicRef(ctx, "HEAD"); err == nil { for _, update := range updates { if update.Dst == localHead && update.NewOID != "" { return localHead } } } return chooseHead(updates) } func (p *Publisher) progress(format string, args ...any) { if p.Progress != nil { p.Progress(format, args...) } } func runParallel(tasks ...func() error) error { if len(tasks) == 0 { return nil } var workers sync.WaitGroup errCh := make(chan error, len(tasks)) for _, task := range tasks { task := task workers.Add(1) go func() { defer workers.Done() if err := task(); err != nil { errCh <- err } }() } workers.Wait() close(errCh) for err := range errCh { return err } return nil }