Commit 539a482

Nick Faro committed on
Publish small pushes as loose Git objects
commit 539a4827b914f67766972f62fa9f6b14d199fc7e parent e462269
3 changed files +147−27
Modifiedinternal/gitutil/git.go +69−0
@@ -100,6 +100,12 @@ type Pack struct {
100 cleanup func() 100 cleanup func()
101 } 101 }
102 102
103 type LooseObject struct {
104 OID string
105 Path string
106 Size int64
107 }
108
103 func (p *Pack) Close() { 109 func (p *Pack) Close() {
104 if p != nil && p.cleanup != nil { 110 if p != nil && p.cleanup != nil {
105 p.cleanup() 111 p.cleanup()
@@ -157,6 +163,69 @@ func (g Git) PackObjects(ctx context.Context, wants, possibleHaves []string) (*P
157 }, nil 163 }, nil
158 } 164 }
159 165
166 // SmallLooseObjects returns the objects represented by a small pack when all
167 // of them still exist in the repository's loose-object store. Recent commits
168 // normally satisfy this, allowing an object-store remote to avoid publishing a
169 // pack index and pack-list update for tiny pushes.
170 func (g Git) SmallLooseObjects(ctx context.Context, pack *Pack, maxObjects int, maxBytes int64) ([]LooseObject, bool, error) {
171 if pack == nil || pack.Objects == 0 || int(pack.Objects) > maxObjects {
172 return nil, false, nil
173 }
174 out, err := g.Run(ctx, "verify-pack", "-v", pack.IndexPath)
175 if err != nil {
176 return nil, false, err
177 }
178 var oids []string
179 for _, line := range strings.Split(string(out), "\n") {
180 fields := strings.Fields(line)
181 if len(fields) < 2 || !isObjectID(fields[0]) {
182 continue
183 }
184 oids = append(oids, fields[0])
185 }
186 if len(oids) != int(pack.Objects) {
187 return nil, false, nil
188 }
189 objectsOutput, err := g.Run(ctx, "rev-parse", "--path-format=absolute", "--git-path", "objects")
190 if err != nil {
191 return nil, false, err
192 }
193 objectsDir := strings.TrimSpace(string(objectsOutput))
194 result := make([]LooseObject, 0, len(oids))
195 var total int64
196 for _, oid := range oids {
197 filename := filepath.Join(objectsDir, oid[:2], oid[2:])
198 info, err := os.Stat(filename)
199 if err != nil {
200 if os.IsNotExist(err) {
201 return nil, false, nil
202 }
203 return nil, false, err
204 }
205 if !info.Mode().IsRegular() {
206 return nil, false, nil
207 }
208 total += info.Size()
209 if total > maxBytes {
210 return nil, false, nil
211 }
212 result = append(result, LooseObject{OID: oid, Path: filename, Size: info.Size()})
213 }
214 return result, true, nil
215 }
216
217 func isObjectID(value string) bool {
218 if len(value) != 40 && len(value) != 64 {
219 return false
220 }
221 for _, char := range []byte(value) {
222 if (char < '0' || char > '9') && (char < 'a' || char > 'f') {
223 return false
224 }
225 }
226 return true
227 }
228
160 func packObjectCount(filename string) (uint32, error) { 229 func packObjectCount(filename string) (uint32, error) {
161 // #nosec G304 -- filename is an internal path in our private temporary directory. 230 // #nosec G304 -- filename is an internal path in our private temporary directory.
162 f, err := os.Open(filename) 231 f, err := os.Open(filename)
Modifiedinternal/gitutil/git_test.go +12−0
@@ -34,6 +34,18 @@ func TestPackObjectsIsIncremental(t *testing.T) {
34 if secondPack.Objects != 3 { 34 if secondPack.Objects != 3 {
35 t.Fatalf("incremental pack has %d objects, want 3", secondPack.Objects) 35 t.Fatalf("incremental pack has %d objects, want 3", secondPack.Objects)
36 } 36 }
37 loose, ok, err := git.SmallLooseObjects(ctx, secondPack, 64, 8<<20)
38 if err != nil {
39 t.Fatal(err)
40 }
41 if !ok || len(loose) != int(secondPack.Objects) {
42 t.Fatalf("SmallLooseObjects() returned %d objects, ok=%v; want %d", len(loose), ok, secondPack.Objects)
43 }
44 for _, object := range loose {
45 if _, err := os.Stat(object.Path); err != nil {
46 t.Fatalf("loose object %s is unavailable: %v", object.OID, err)
47 }
48 }
37 } 49 }
38 50
39 func initTestRepository(t *testing.T) string { 51 func initTestRepository(t *testing.T) string {
Modifiedinternal/publish/publish.go +66−27
@@ -148,24 +148,61 @@ func (p *Publisher) Push(ctx context.Context, updates []Update, expected map[str
148 } 148 }
149 defer pack.Close() 149 defer pack.Close()
150 packAdded := false 150 packAdded := false
151 refsPublishedEarly := false
151 if pack != nil && pack.Objects > 0 { 152 if pack != nil && pack.Objects > 0 {
152 packName := "pack-" + pack.Hash + ".pack" 153 looseObjects, useLoose, looseErr := p.Git.SmallLooseObjects(ctx, pack, 64, 8<<20)
153 indexName := "pack-" + pack.Hash + ".idx" 154 if looseErr != nil {
154 p.progress("uploading %s (%d objects)", packName, pack.Objects) 155 return Result{}, looseErr
155 if err := runParallel( 156 }
156 func() error { 157 if useLoose {
157 return p.Store.PutFile(ctx, path.Join("objects/pack", packName), pack.PackPath, "application/x-git-packed-objects", immutableCache) 158 p.progress("uploading %d loose objects", len(looseObjects))
158 }, 159 publications := make([]func() error, 0, len(looseObjects)+len(resolved))
159 func() error { 160 for _, object := range looseObjects {
160 return p.Store.PutFile(ctx, path.Join("objects/pack", indexName), pack.IndexPath, "application/x-git-packed-objects-toc", immutableCache) 161 object := object
161 }, 162 publications = append(publications, func() error {
162 ); err != nil { 163 return p.Store.PutFile(ctx, path.Join("objects", object.OID[:2], object.OID[2:]), object.Path, "application/x-git-loose-object", immutableCache)
163 return Result{}, err 164 })
165 }
166 // Individual ref files are not used for discovery by dumb HTTP; publish
167 // them in the same wave and update the authoritative info/refs only
168 // after every object and ref write has succeeded.
169 for _, update := range resolved {
170 update := update
171 if update.OldOID == update.NewOID {
172 continue
173 }
174 if update.NewOID == "" {
175 publications = append(publications, func() error { return p.Store.Delete(ctx, update.Dst) })
176 } else {
177 publications = append(publications, func() error {
178 return p.Store.Put(ctx, update.Dst, []byte(update.NewOID+"\n"), "text/plain; charset=utf-8", mutableCache)
179 })
180 }
181 }
182 if err := runParallel(publications...); err != nil {
183 return Result{}, err
184 }
185 refsPublishedEarly = true
186 result.PackObjects = pack.Objects
187 } else {
188 packName := "pack-" + pack.Hash + ".pack"
189 indexName := "pack-" + pack.Hash + ".idx"
190 p.progress("uploading %s (%d objects)", packName, pack.Objects)
191 if err := runParallel(
192 func() error {
193 return p.Store.PutFile(ctx, path.Join("objects/pack", packName), pack.PackPath, "application/x-git-packed-objects", immutableCache)
194 },
195 func() error {
196 return p.Store.PutFile(ctx, path.Join("objects/pack", indexName), pack.IndexPath, "application/x-git-packed-objects-toc", immutableCache)
197 },
198 ); err != nil {
199 return Result{}, err
200 }
201 metadata.Packs[packName] = struct{}{}
202 result.PackName = packName
203 result.PackObjects = pack.Objects
204 packAdded = true
164 } 205 }
165 metadata.Packs[packName] = struct{}{}
166 result.PackName = packName
167 result.PackObjects = pack.Objects
168 packAdded = true
169 } 206 }
170 207
171 previousHead := metadata.Head 208 previousHead := metadata.Head
@@ -203,18 +240,20 @@ func (p *Publisher) Push(ctx context.Context, updates []Update, expected map[str
203 return p.Store.Put(ctx, "objects/info/packs", renderPackList(metadata.Packs), "text/plain; charset=utf-8", mutableCache) 240 return p.Store.Put(ctx, "objects/info/packs", renderPackList(metadata.Packs), "text/plain; charset=utf-8", mutableCache)
204 }) 241 })
205 } 242 }
206 for _, update := range resolved { 243 if !refsPublishedEarly {
207 update := update 244 for _, update := range resolved {
208 if update.OldOID == update.NewOID { 245 update := update
209 continue 246 if update.OldOID == update.NewOID {
210 } 247 continue
211 if update.NewOID == "" { 248 }
212 publications = append(publications, func() error { return p.Store.Delete(ctx, update.Dst) }) 249 if update.NewOID == "" {
213 continue 250 publications = append(publications, func() error { return p.Store.Delete(ctx, update.Dst) })
251 continue
252 }
253 publications = append(publications, func() error {
254 return p.Store.Put(ctx, update.Dst, []byte(update.NewOID+"\n"), "text/plain; charset=utf-8", mutableCache)
255 })
214 } 256 }
215 publications = append(publications, func() error {
216 return p.Store.Put(ctx, update.Dst, []byte(update.NewOID+"\n"), "text/plain; charset=utf-8", mutableCache)
217 })
218 } 257 }
219 if metadata.Head != "" && metadata.Head != previousHead { 258 if metadata.Head != "" && metadata.Head != previousHead {
220 publications = append(publications, func() error { 259 publications = append(publications, func() error {