Commit 989ca7f

Nick Faro committed on
Speed up R2 publication
commit 989ca7fd895146085c16c28493fab6f4cd5fbec0 parent 4bcc586
6 changed files +239−60
Modifiedinternal/helper/helper.go +13−6
@@ -139,6 +139,7 @@ func publishSite(ctx context.Context, publisher *publish.Publisher, result publi
139 } 139 }
140 generator.SetBranches(result.Branches, strings.TrimPrefix(result.Head, "refs/heads/")) 140 generator.SetBranches(result.Branches, strings.TrimPrefix(result.Head, "refs/heads/"))
141 var deletedBranches []string 141 var deletedBranches []string
142 navigationChanged := false
142 for _, update := range result.Updates { 143 for _, update := range result.Updates {
143 if !strings.HasPrefix(update.Dst, "refs/heads/") { 144 if !strings.HasPrefix(update.Dst, "refs/heads/") {
144 continue 145 continue
@@ -146,8 +147,12 @@ func publishSite(ctx context.Context, publisher *publish.Publisher, result publi
146 branch := strings.TrimPrefix(update.Dst, "refs/heads/") 147 branch := strings.TrimPrefix(update.Dst, "refs/heads/")
147 if update.NewOID == "" { 148 if update.NewOID == "" {
148 deletedBranches = append(deletedBranches, branch) 149 deletedBranches = append(deletedBranches, branch)
150 navigationChanged = true
149 continue 151 continue
150 } 152 }
153 if update.OldOID == "" {
154 navigationChanged = true
155 }
151 build, err := generator.BuildBranch(ctx, branch, update.OldOID, update.NewOID, update.Dst == result.Head) 156 build, err := generator.BuildBranch(ctx, branch, update.OldOID, update.NewOID, update.Dst == result.Head)
152 if err == nil { 157 if err == nil {
153 err = generator.Publish(ctx, publisher.Store, build) 158 err = generator.Publish(ctx, publisher.Store, build)
@@ -156,12 +161,14 @@ func publishSite(ctx context.Context, publisher *publish.Publisher, result publi
156 fmt.Fprintf(stderr, "dumbforge: warning: site publication failed: %v\n", err) 161 fmt.Fprintf(stderr, "dumbforge: warning: site publication failed: %v\n", err)
157 } 162 }
158 } 163 }
159 navigation, err := generator.RepositoryNavigationBuild(deletedBranches) 164 if navigationChanged {
160 if err == nil { 165 navigation, err := generator.RepositoryNavigationBuild(deletedBranches)
161 err = generator.Publish(ctx, publisher.Store, navigation) 166 if err == nil {
162 } 167 err = generator.Publish(ctx, publisher.Store, navigation)
163 if err != nil { 168 }
164 fmt.Fprintf(stderr, "dumbforge: warning: repository navigation publication failed: %v\n", err) 169 if err != nil {
170 fmt.Fprintf(stderr, "dumbforge: warning: repository navigation publication failed: %v\n", err)
171 }
165 } 172 }
166 } 173 }
167 174
Modifiedinternal/publish/metadata.go +22−3
@@ -7,6 +7,7 @@ import (
7 "fmt" 7 "fmt"
8 "sort" 8 "sort"
9 "strings" 9 "strings"
10 "sync"
10 11
11 "github.com/npfaro/dumbforge/internal/s3store" 12 "github.com/npfaro/dumbforge/internal/s3store"
12 ) 13 )
@@ -38,7 +39,25 @@ func LoadMetadata(ctx context.Context, store *s3store.Store) (Metadata, error) {
38 Refs: map[string]Ref{}, 39 Refs: map[string]Ref{},
39 Packs: map[string]struct{}{}, 40 Packs: map[string]struct{}{},
40 } 41 }
41 refsObject, err := store.Get(ctx, "info/refs") 42 var refsObject, headObject, packObject s3store.Object
43 var refsErr, headErr, packErr error
44 var requests sync.WaitGroup
45 requests.Add(3)
46 go func() {
47 defer requests.Done()
48 refsObject, refsErr = store.Get(ctx, "info/refs")
49 }()
50 go func() {
51 defer requests.Done()
52 headObject, headErr = store.Get(ctx, "HEAD")
53 }()
54 go func() {
55 defer requests.Done()
56 packObject, packErr = store.Get(ctx, "objects/info/packs")
57 }()
58 requests.Wait()
59
60 err := refsErr
42 if err != nil && !s3store.IsNotFound(err) { 61 if err != nil && !s3store.IsNotFound(err) {
43 return Metadata{}, fmt.Errorf("read remote refs: %w", err) 62 return Metadata{}, fmt.Errorf("read remote refs: %w", err)
44 } 63 }
@@ -47,7 +66,7 @@ func LoadMetadata(ctx context.Context, store *s3store.Store) (Metadata, error) {
47 return Metadata{}, err 66 return Metadata{}, err
48 } 67 }
49 } 68 }
50 headObject, err := store.Get(ctx, "HEAD") 69 err = headErr
51 if err != nil && !s3store.IsNotFound(err) { 70 if err != nil && !s3store.IsNotFound(err) {
52 return Metadata{}, fmt.Errorf("read remote HEAD: %w", err) 71 return Metadata{}, fmt.Errorf("read remote HEAD: %w", err)
53 } 72 }
@@ -57,7 +76,7 @@ func LoadMetadata(ctx context.Context, store *s3store.Store) (Metadata, error) {
57 metadata.Head = strings.TrimSpace(strings.TrimPrefix(line, "ref: ")) 76 metadata.Head = strings.TrimSpace(strings.TrimPrefix(line, "ref: "))
58 } 77 }
59 } 78 }
60 packObject, err := store.Get(ctx, "objects/info/packs") 79 err = packErr
61 if err != nil && !s3store.IsNotFound(err) { 80 if err != nil && !s3store.IsNotFound(err) {
62 return Metadata{}, fmt.Errorf("read remote pack list: %w", err) 81 return Metadata{}, fmt.Errorf("read remote pack list: %w", err)
63 } 82 }
Modifiedinternal/publish/publish.go +61−19
@@ -6,6 +6,8 @@ import (
6 "path" 6 "path"
7 "sort" 7 "sort"
8 "strings" 8 "strings"
9 "sync"
10 "time"
9 11
10 "github.com/npfaro/dumbforge/internal/gitutil" 12 "github.com/npfaro/dumbforge/internal/gitutil"
11 "github.com/npfaro/dumbforge/internal/s3store" 13 "github.com/npfaro/dumbforge/internal/s3store"
@@ -98,21 +100,28 @@ func (p *Publisher) Push(ctx context.Context, updates []Update, expected map[str
98 return Result{}, err 100 return Result{}, err
99 } 101 }
100 defer pack.Close() 102 defer pack.Close()
103 packAdded := false
101 if pack != nil && pack.Objects > 0 { 104 if pack != nil && pack.Objects > 0 {
102 packName := "pack-" + pack.Hash + ".pack" 105 packName := "pack-" + pack.Hash + ".pack"
103 indexName := "pack-" + pack.Hash + ".idx" 106 indexName := "pack-" + pack.Hash + ".idx"
104 p.progress("uploading %s (%d objects)", packName, pack.Objects) 107 p.progress("uploading %s (%d objects)", packName, pack.Objects)
105 if err := p.Store.PutFile(ctx, path.Join("objects/pack", packName), pack.PackPath, "application/x-git-packed-objects", immutableCache); err != nil { 108 if err := runParallel(
106 return Result{}, err 109 func() error {
107 } 110 return p.Store.PutFile(ctx, path.Join("objects/pack", packName), pack.PackPath, "application/x-git-packed-objects", immutableCache)
108 if err := p.Store.PutFile(ctx, path.Join("objects/pack", indexName), pack.IndexPath, "application/x-git-packed-objects-toc", immutableCache); err != nil { 111 },
112 func() error {
113 return p.Store.PutFile(ctx, path.Join("objects/pack", indexName), pack.IndexPath, "application/x-git-packed-objects-toc", immutableCache)
114 },
115 ); err != nil {
109 return Result{}, err 116 return Result{}, err
110 } 117 }
111 metadata.Packs[packName] = struct{}{} 118 metadata.Packs[packName] = struct{}{}
112 result.PackName = packName 119 result.PackName = packName
113 result.PackObjects = pack.Objects 120 result.PackObjects = pack.Objects
121 packAdded = true
114 } 122 }
115 123
124 previousHead := metadata.Head
116 for _, update := range resolved { 125 for _, update := range resolved {
117 result.Updates = append(result.Updates, update) 126 result.Updates = append(result.Updates, update)
118 if update.NewOID == "" { 127 if update.NewOID == "" {
@@ -133,30 +142,39 @@ func (p *Publisher) Push(ctx context.Context, updates []Update, expected map[str
133 } 142 }
134 result.Head = metadata.Head 143 result.Head = metadata.Head
135 result.Branches = metadata.BranchNames() 144 result.Branches = metadata.BranchNames()
136 if err := lock.Refresh(ctx); err != nil { 145 if err := lock.RefreshIfOlderThan(ctx, 30*time.Minute); err != nil {
137 return Result{}, err 146 return Result{}, err
138 } 147 }
139 148
140 // Publish packs first, then individual refs and HEAD, and info/refs last. 149 // Packs are already durable. Publish the remaining internal pointers in
141 // A dumb-HTTP reader can therefore never discover a ref before its pack. 150 // parallel, then info/refs last so dumb-HTTP clients cannot discover a ref
142 if err := p.Store.Put(ctx, "objects/info/packs", renderPackList(metadata.Packs), "text/plain; charset=utf-8", mutableCache); err != nil { 151 // before all of the objects needed to read it are available.
143 return Result{}, err 152 var publications []func() error
153 if packAdded {
154 publications = append(publications, func() error {
155 return p.Store.Put(ctx, "objects/info/packs", renderPackList(metadata.Packs), "text/plain; charset=utf-8", mutableCache)
156 })
144 } 157 }
145 for _, update := range resolved { 158 for _, update := range resolved {
146 if update.NewOID == "" { 159 update := update
147 if err := p.Store.Delete(ctx, update.Dst); err != nil { 160 if update.OldOID == update.NewOID {
148 return Result{}, err
149 }
150 continue 161 continue
151 } 162 }
152 if err := p.Store.Put(ctx, update.Dst, []byte(update.NewOID+"\n"), "text/plain; charset=utf-8", mutableCache); err != nil { 163 if update.NewOID == "" {
153 return Result{}, err 164 publications = append(publications, func() error { return p.Store.Delete(ctx, update.Dst) })
165 continue
154 } 166 }
167 publications = append(publications, func() error {
168 return p.Store.Put(ctx, update.Dst, []byte(update.NewOID+"\n"), "text/plain; charset=utf-8", mutableCache)
169 })
155 } 170 }
156 if metadata.Head != "" { 171 if metadata.Head != "" && metadata.Head != previousHead {
157 if err := p.Store.Put(ctx, "HEAD", []byte("ref: "+metadata.Head+"\n"), "text/plain; charset=utf-8", mutableCache); err != nil { 172 publications = append(publications, func() error {
158 return Result{}, err 173 return p.Store.Put(ctx, "HEAD", []byte("ref: "+metadata.Head+"\n"), "text/plain; charset=utf-8", mutableCache)
159 } 174 })
175 }
176 if err := runParallel(publications...); err != nil {
177 return Result{}, err
160 } 178 }
161 if err := p.Store.Put(ctx, "info/refs", renderInfoRefs(metadata.Refs), "text/plain; charset=utf-8", mutableCache); err != nil { 179 if err := p.Store.Put(ctx, "info/refs", renderInfoRefs(metadata.Refs), "text/plain; charset=utf-8", mutableCache); err != nil {
162 return Result{}, err 180 return Result{}, err
@@ -253,3 +271,27 @@ func (p *Publisher) progress(format string, args ...any) {
253 p.Progress(format, args...) 271 p.Progress(format, args...)
254 } 272 }
255 } 273 }
274
275 func runParallel(tasks ...func() error) error {
276 if len(tasks) == 0 {
277 return nil
278 }
279 var workers sync.WaitGroup
280 errCh := make(chan error, len(tasks))
281 for _, task := range tasks {
282 task := task
283 workers.Add(1)
284 go func() {
285 defer workers.Done()
286 if err := task(); err != nil {
287 errCh <- err
288 }
289 }()
290 }
291 workers.Wait()
292 close(errCh)
293 for err := range errCh {
294 return err
295 }
296 return nil
297 }
Modifiedinternal/s3store/store.go +37−21
@@ -42,6 +42,9 @@ func New(ctx context.Context, spec remoteurl.Spec) (*Store, error) {
42 client := s3.NewFromConfig(cfg, func(o *s3.Options) { 42 client := s3.NewFromConfig(cfg, func(o *s3.Options) {
43 o.BaseEndpoint = aws.String(spec.Endpoint) 43 o.BaseEndpoint = aws.String(spec.Endpoint)
44 o.UsePathStyle = true 44 o.UsePathStyle = true
45 // S3-compatible services do not uniformly support the SDK's optional
46 // streaming CRC32 trailer, especially when Content-Encoding is set.
47 o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
45 }) 48 })
46 return &Store{client: client, spec: spec}, nil 49 return &Store{client: client, spec: spec}, nil
47 } 50 }
@@ -74,35 +77,37 @@ func (s *Store) Get(ctx context.Context, name string) (Object, error) {
74 } 77 }
75 78
76 func (s *Store) Put(ctx context.Context, name string, data []byte, contentType, cacheControl string) error { 79 func (s *Store) Put(ctx context.Context, name string, data []byte, contentType, cacheControl string) error {
77 input := &s3.PutObjectInput{ 80 return s.putBytes(ctx, s.key(name), data, contentType, cacheControl, "")
78 Bucket: aws.String(s.spec.Bucket), 81 }
79 Key: aws.String(s.key(name)), 82
80 Body: bytes.NewReader(data), 83 func (s *Store) PutEncoded(ctx context.Context, name string, data []byte, contentType, cacheControl, contentEncoding string) error {
81 ContentType: aws.String(contentType), 84 return s.putBytes(ctx, s.key(name), data, contentType, cacheControl, contentEncoding)
82 }
83 if cacheControl != "" {
84 input.CacheControl = aws.String(cacheControl)
85 }
86 _, err := s.client.PutObject(ctx, input)
87 if err != nil {
88 return fmt.Errorf("put s3://%s/%s: %w", s.spec.Bucket, s.key(name), err)
89 }
90 return nil
91 } 85 }
92 86
93 func (s *Store) PutRoot(ctx context.Context, data []byte, contentType, cacheControl string) error { 87 func (s *Store) PutRoot(ctx context.Context, data []byte, contentType, cacheControl string) error {
88 return s.putBytes(ctx, s.RootKey(), data, contentType, cacheControl, "")
89 }
90
91 func (s *Store) PutRootEncoded(ctx context.Context, data []byte, contentType, cacheControl, contentEncoding string) error {
92 return s.putBytes(ctx, s.RootKey(), data, contentType, cacheControl, contentEncoding)
93 }
94
95 func (s *Store) putBytes(ctx context.Context, key string, data []byte, contentType, cacheControl, contentEncoding string) error {
94 input := &s3.PutObjectInput{ 96 input := &s3.PutObjectInput{
95 Bucket: aws.String(s.spec.Bucket), 97 Bucket: aws.String(s.spec.Bucket),
96 Key: aws.String(s.RootKey()), 98 Key: aws.String(key),
97 Body: bytes.NewReader(data), 99 Body: bytes.NewReader(data),
98 ContentType: aws.String(contentType), 100 ContentType: aws.String(contentType),
99 } 101 }
100 if cacheControl != "" { 102 if cacheControl != "" {
101 input.CacheControl = aws.String(cacheControl) 103 input.CacheControl = aws.String(cacheControl)
102 } 104 }
105 if contentEncoding != "" {
106 input.ContentEncoding = aws.String(contentEncoding)
107 }
103 _, err := s.client.PutObject(ctx, input) 108 _, err := s.client.PutObject(ctx, input)
104 if err != nil { 109 if err != nil {
105 return fmt.Errorf("put s3://%s/%s: %w", s.spec.Bucket, s.RootKey(), err) 110 return fmt.Errorf("put s3://%s/%s: %w", s.spec.Bucket, key, err)
106 } 111 }
107 return nil 112 return nil
108 } 113 }
@@ -142,10 +147,11 @@ func (s *Store) Delete(ctx context.Context, name string) error {
142 } 147 }
143 148
144 type Lock struct { 149 type Lock struct {
145 store *Store 150 store *Store
146 name string 151 name string
147 token string 152 token string
148 etag string 153 etag string
154 refreshedAt time.Time
149 } 155 }
150 156
151 func (s *Store) AcquireLock(ctx context.Context) (*Lock, error) { 157 func (s *Store) AcquireLock(ctx context.Context) (*Lock, error) {
@@ -188,7 +194,7 @@ func (s *Store) AcquireLock(ctx context.Context) (*Lock, error) {
188 return nil, fmt.Errorf("acquire repository lock: %w", err) 194 return nil, fmt.Errorf("acquire repository lock: %w", err)
189 } 195 }
190 } 196 }
191 return &Lock{store: s, name: name, token: token, etag: aws.ToString(out.ETag)}, nil 197 return &Lock{store: s, name: name, token: token, etag: aws.ToString(out.ETag), refreshedAt: time.Now()}, nil
192 } 198 }
193 199
194 // Refresh proves that this process still owns the lease before it publishes 200 // Refresh proves that this process still owns the lease before it publishes
@@ -206,9 +212,19 @@ func (l *Lock) Refresh(ctx context.Context) error {
206 return fmt.Errorf("refresh repository lock: %w", err) 212 return fmt.Errorf("refresh repository lock: %w", err)
207 } 213 }
208 l.etag = aws.ToString(out.ETag) 214 l.etag = aws.ToString(out.ETag)
215 l.refreshedAt = time.Now()
209 return nil 216 return nil
210 } 217 }
211 218
219 // RefreshIfOlderThan avoids an S3 round trip for ordinary pushes while still
220 // renewing the lease during unusually long publications.
221 func (l *Lock) RefreshIfOlderThan(ctx context.Context, age time.Duration) error {
222 if time.Since(l.refreshedAt) < age {
223 return nil
224 }
225 return l.Refresh(ctx)
226 }
227
212 func (l *Lock) Release(ctx context.Context) error { 228 func (l *Lock) Release(ctx context.Context) error {
213 _, err := l.store.client.DeleteObject(ctx, &s3.DeleteObjectInput{ 229 _, err := l.store.client.DeleteObject(ctx, &s3.DeleteObjectInput{
214 Bucket: aws.String(l.store.spec.Bucket), 230 Bucket: aws.String(l.store.spec.Bucket),
Modifiedinternal/site/site.go +61−9
@@ -2,6 +2,7 @@ package site
2 2
3 import ( 3 import (
4 "bytes" 4 "bytes"
5 "compress/gzip"
5 "context" 6 "context"
6 "fmt" 7 "fmt"
7 "html/template" 8 "html/template"
@@ -170,13 +171,13 @@ func (g *Generator) BuildBranch(ctx context.Context, branch, oldOID, newOID stri
170 return Build{}, err 171 return Build{}, err
171 } 172 }
172 result.Pages = append(result.Pages, commitsPage) 173 result.Pages = append(result.Pages, commitsPage)
173 filesPage, err := g.filesPage(ctx, branch, newOID)
174 if err != nil {
175 return Build{}, err
176 }
177 result.Pages = append(result.Pages, filesPage)
178 174
179 if oldOID == "" { 175 if oldOID == "" {
176 filesPage, err := g.filesPage(ctx, branch, newOID)
177 if err != nil {
178 return Build{}, err
179 }
180 result.Pages = append(result.Pages, filesPage)
180 entries, err := g.listTreeRecursive(ctx, newOID) 181 entries, err := g.listTreeRecursive(ctx, newOID)
181 if err != nil { 182 if err != nil {
182 return Build{}, err 183 return Build{}, err
@@ -210,6 +211,13 @@ func (g *Generator) BuildBranch(ctx context.Context, branch, oldOID, newOID stri
210 g.progress("could not diff %s..%s; rebuilding branch", short(oldOID), short(newOID)) 211 g.progress("could not diff %s..%s; rebuilding branch", short(oldOID), short(newOID))
211 return g.BuildBranch(ctx, branch, "", newOID, isHead) 212 return g.BuildBranch(ctx, branch, "", newOID, isHead)
212 } 213 }
214 if fileNavigationChanged(changes) {
215 filesPage, err := g.filesPage(ctx, branch, newOID)
216 if err != nil {
217 return Build{}, err
218 }
219 result.Pages = append(result.Pages, filesPage)
220 }
213 dirs := map[string]struct{}{"": {}} 221 dirs := map[string]struct{}{"": {}}
214 for _, change := range changes { 222 for _, change := range changes {
215 if change.OldPath != "" { 223 if change.OldPath != "" {
@@ -271,6 +279,15 @@ func (g *Generator) BuildBranch(ctx context.Context, branch, oldOID, newOID stri
271 return result, nil 279 return result, nil
272 } 280 }
273 281
282 func fileNavigationChanged(changes []gitutil.NameStatus) bool {
283 for _, change := range changes {
284 if change.Status == "" || change.Status[0] != 'M' || change.Path == ".gitmodules" || change.OldPath == ".gitmodules" {
285 return true
286 }
287 }
288 return false
289 }
290
274 func (g *Generator) Publish(ctx context.Context, store *s3store.Store, build Build) error { 291 func (g *Generator) Publish(ctx context.Context, store *s3store.Store, build Build) error {
275 for _, route := range build.Deletes { 292 for _, route := range build.Deletes {
276 if err := store.Delete(ctx, route); err != nil { 293 if err := store.Delete(ctx, route); err != nil {
@@ -284,16 +301,23 @@ func (g *Generator) Publish(ctx context.Context, store *s3store.Store, build Bui
284 errCh := make(chan error, 1) 301 errCh := make(chan error, 1)
285 var once sync.Once 302 var once sync.Once
286 var workers sync.WaitGroup 303 var workers sync.WaitGroup
287 for range 8 { 304 for range 32 {
288 workers.Add(1) 305 workers.Add(1)
289 go func() { 306 go func() {
290 defer workers.Done() 307 defer workers.Done()
291 for page := range jobs { 308 for page := range jobs {
292 var err error 309 body, encoding, err := compressedPage(page)
310 if err != nil {
311 once.Do(func() {
312 errCh <- err
313 cancel()
314 })
315 return
316 }
293 if page.Root { 317 if page.Root {
294 err = store.PutRoot(ctx, page.Body, page.ContentType, page.Cache) 318 err = store.PutRootEncoded(ctx, body, page.ContentType, page.Cache, encoding)
295 } else { 319 } else {
296 err = store.Put(ctx, page.Route, page.Body, page.ContentType, page.Cache) 320 err = store.PutEncoded(ctx, page.Route, body, page.ContentType, page.Cache, encoding)
297 } 321 }
298 if err != nil { 322 if err != nil {
299 once.Do(func() { 323 once.Do(func() {
@@ -324,6 +348,34 @@ sendPages:
324 return nil 348 return nil
325 } 349 }
326 350
351 func compressedPage(page Page) ([]byte, string, error) {
352 if len(page.Body) < 1024 || !compressibleContentType(page.ContentType) {
353 return page.Body, "", nil
354 }
355 var compressed bytes.Buffer
356 writer, err := gzip.NewWriterLevel(&compressed, gzip.BestSpeed)
357 if err != nil {
358 return nil, "", err
359 }
360 if _, err := writer.Write(page.Body); err != nil {
361 return nil, "", err
362 }
363 if err := writer.Close(); err != nil {
364 return nil, "", err
365 }
366 if compressed.Len() >= len(page.Body) {
367 return page.Body, "", nil
368 }
369 return compressed.Bytes(), "gzip", nil
370 }
371
372 func compressibleContentType(contentType string) bool {
373 return strings.HasPrefix(contentType, "text/") ||
374 strings.HasPrefix(contentType, "application/json") ||
375 strings.HasPrefix(contentType, "application/javascript") ||
376 strings.HasPrefix(contentType, "image/svg+xml")
377 }
378
327 func (g *Generator) treePage(ctx context.Context, branch, oid, dir string, summary branchSummary, root bool) (Page, error) { 379 func (g *Generator) treePage(ctx context.Context, branch, oid, dir string, summary branchSummary, root bool) (Page, error) {
328 entries, err := g.listTree(ctx, oid, dir) 380 entries, err := g.listTree(ctx, oid, dir)
329 if err != nil { 381 if err != nil {
Modifiedinternal/site/site_test.go +45−2
@@ -1,7 +1,10 @@
1 package site 1 package site
2 2
3 import ( 3 import (
4 "bytes"
5 "compress/gzip"
4 "context" 6 "context"
7 "io"
5 "os" 8 "os"
6 "os/exec" 9 "os/exec"
7 "path/filepath" 10 "path/filepath"
@@ -83,8 +86,8 @@ func TestFullAndIncrementalBuild(t *testing.T) {
83 if err != nil { 86 if err != nil {
84 t.Fatal(err) 87 t.Fatal(err)
85 } 88 }
86 if len(incremental.Pages) != 9 { 89 if len(incremental.Pages) != 8 {
87 t.Fatalf("incremental build generated %d pages, want 9", len(incremental.Pages)) 90 t.Fatalf("incremental build generated %d pages, want 8", len(incremental.Pages))
88 } 91 }
89 assertPage(t, incremental, "blob/main/src/main.go", "changed") 92 assertPage(t, incremental, "blob/main/src/main.go", "changed")
90 assertPage(t, incremental, "tree/main/src", "change one file") 93 assertPage(t, incremental, "tree/main/src", "change one file")
@@ -212,6 +215,46 @@ func TestGravatarUsesNormalizedSHA256(t *testing.T) {
212 } 215 }
213 } 216 }
214 217
218 func TestCompressedPageRoundTrip(t *testing.T) {
219 body := bytes.Repeat([]byte("compress me\n"), 1000)
220 compressed, encoding, err := compressedPage(Page{Body: body, ContentType: "text/html; charset=utf-8"})
221 if err != nil {
222 t.Fatal(err)
223 }
224 if encoding != "gzip" || len(compressed) >= len(body) {
225 t.Fatalf("compression = %q and %d bytes; want gzip smaller than %d", encoding, len(compressed), len(body))
226 }
227 reader, err := gzip.NewReader(bytes.NewReader(compressed))
228 if err != nil {
229 t.Fatal(err)
230 }
231 decoded, err := io.ReadAll(reader)
232 if err != nil {
233 t.Fatal(err)
234 }
235 if err := reader.Close(); err != nil {
236 t.Fatal(err)
237 }
238 if !bytes.Equal(decoded, body) {
239 t.Fatal("compressed page did not round-trip")
240 }
241 }
242
243 func TestFileNavigationChanged(t *testing.T) {
244 if fileNavigationChanged([]gitutil.NameStatus{{Status: "M", Path: "src/main.go"}}) {
245 t.Fatal("content-only modification should not rebuild file navigation")
246 }
247 for _, change := range []gitutil.NameStatus{
248 {Status: "A", Path: "new.go"},
249 {Status: "D", Path: "old.go"},
250 {Status: "M", Path: ".gitmodules"},
251 } {
252 if !fileNavigationChanged([]gitutil.NameStatus{change}) {
253 t.Fatalf("%+v should rebuild file navigation", change)
254 }
255 }
256 }
257
215 func assertPage(t *testing.T, build Build, route, contains string) { 258 func assertPage(t *testing.T, build Build, route, contains string) {
216 t.Helper() 259 t.Helper()
217 for _, page := range build.Pages { 260 for _, page := range build.Pages {