1
2
3
4
5 package modfetch
6
7 import (
8 "bytes"
9 "encoding/json"
10 "errors"
11 "fmt"
12 "io"
13 "io/fs"
14 "math/rand"
15 "os"
16 "path/filepath"
17 "strconv"
18 "strings"
19 "sync"
20
21 "cmd/go/internal/base"
22 "cmd/go/internal/cfg"
23 "cmd/go/internal/lockedfile"
24 "cmd/go/internal/modfetch/codehost"
25 "cmd/go/internal/par"
26 "cmd/go/internal/robustio"
27
28 "golang.org/x/mod/module"
29 "golang.org/x/mod/semver"
30 )
31
32 func cacheDir(path string) (string, error) {
33 if err := checkCacheDir(); err != nil {
34 return "", err
35 }
36 enc, err := module.EscapePath(path)
37 if err != nil {
38 return "", err
39 }
40 return filepath.Join(cfg.GOMODCACHE, "cache/download", enc, "/@v"), nil
41 }
42
43 func CachePath(m module.Version, suffix string) (string, error) {
44 dir, err := cacheDir(m.Path)
45 if err != nil {
46 return "", err
47 }
48 if !semver.IsValid(m.Version) {
49 return "", fmt.Errorf("non-semver module version %q", m.Version)
50 }
51 if module.CanonicalVersion(m.Version) != m.Version {
52 return "", fmt.Errorf("non-canonical module version %q", m.Version)
53 }
54 encVer, err := module.EscapeVersion(m.Version)
55 if err != nil {
56 return "", err
57 }
58 return filepath.Join(dir, encVer+"."+suffix), nil
59 }
60
61
62
63
64
65
66 func DownloadDir(m module.Version) (string, error) {
67 if err := checkCacheDir(); err != nil {
68 return "", err
69 }
70 enc, err := module.EscapePath(m.Path)
71 if err != nil {
72 return "", err
73 }
74 if !semver.IsValid(m.Version) {
75 return "", fmt.Errorf("non-semver module version %q", m.Version)
76 }
77 if module.CanonicalVersion(m.Version) != m.Version {
78 return "", fmt.Errorf("non-canonical module version %q", m.Version)
79 }
80 encVer, err := module.EscapeVersion(m.Version)
81 if err != nil {
82 return "", err
83 }
84
85
86 dir := filepath.Join(cfg.GOMODCACHE, enc+"@"+encVer)
87 if fi, err := os.Stat(dir); os.IsNotExist(err) {
88 return dir, err
89 } else if err != nil {
90 return dir, &DownloadDirPartialError{dir, err}
91 } else if !fi.IsDir() {
92 return dir, &DownloadDirPartialError{dir, errors.New("not a directory")}
93 }
94
95
96
97 partialPath, err := CachePath(m, "partial")
98 if err != nil {
99 return dir, err
100 }
101 if _, err := os.Stat(partialPath); err == nil {
102 return dir, &DownloadDirPartialError{dir, errors.New("not completely extracted")}
103 } else if !os.IsNotExist(err) {
104 return dir, err
105 }
106
107
108
109
110
111
112 ziphashPath, err := CachePath(m, "ziphash")
113 if err != nil {
114 return dir, err
115 }
116 if _, err := os.Stat(ziphashPath); os.IsNotExist(err) {
117 return dir, &DownloadDirPartialError{dir, errors.New("ziphash file is missing")}
118 } else if err != nil {
119 return dir, err
120 }
121 return dir, nil
122 }
123
124
125
126
127
128 type DownloadDirPartialError struct {
129 Dir string
130 Err error
131 }
132
133 func (e *DownloadDirPartialError) Error() string { return fmt.Sprintf("%s: %v", e.Dir, e.Err) }
134 func (e *DownloadDirPartialError) Is(err error) bool { return err == fs.ErrNotExist }
135
136
137
138 func lockVersion(mod module.Version) (unlock func(), err error) {
139 path, err := CachePath(mod, "lock")
140 if err != nil {
141 return nil, err
142 }
143 if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
144 return nil, err
145 }
146 return lockedfile.MutexAt(path).Lock()
147 }
148
149
150
151
152
153 func SideLock() (unlock func(), err error) {
154 if err := checkCacheDir(); err != nil {
155 return nil, err
156 }
157
158 path := filepath.Join(cfg.GOMODCACHE, "cache", "lock")
159 if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
160 return nil, fmt.Errorf("failed to create cache directory: %w", err)
161 }
162
163 return lockedfile.MutexAt(path).Lock()
164 }
165
166
167
168
169
170
171 type cachingRepo struct {
172 path string
173 cache par.Cache
174
175 once sync.Once
176 initRepo func() (Repo, error)
177 r Repo
178 }
179
180 func newCachingRepo(path string, initRepo func() (Repo, error)) *cachingRepo {
181 return &cachingRepo{
182 path: path,
183 initRepo: initRepo,
184 }
185 }
186
187 func (r *cachingRepo) repo() Repo {
188 r.once.Do(func() {
189 var err error
190 r.r, err = r.initRepo()
191 if err != nil {
192 r.r = errRepo{r.path, err}
193 }
194 })
195 return r.r
196 }
197
198 func (r *cachingRepo) ModulePath() string {
199 return r.path
200 }
201
202 func (r *cachingRepo) Versions(prefix string) ([]string, error) {
203 type cached struct {
204 list []string
205 err error
206 }
207 c := r.cache.Do("versions:"+prefix, func() any {
208 list, err := r.repo().Versions(prefix)
209 return cached{list, err}
210 }).(cached)
211
212 if c.err != nil {
213 return nil, c.err
214 }
215 return append([]string(nil), c.list...), nil
216 }
217
218 type cachedInfo struct {
219 info *RevInfo
220 err error
221 }
222
223 func (r *cachingRepo) Stat(rev string) (*RevInfo, error) {
224 c := r.cache.Do("stat:"+rev, func() any {
225 file, info, err := readDiskStat(r.path, rev)
226 if err == nil {
227 return cachedInfo{info, nil}
228 }
229
230 info, err = r.repo().Stat(rev)
231 if err == nil {
232
233
234 if info.Version != rev {
235 file, _ = CachePath(module.Version{Path: r.path, Version: info.Version}, "info")
236 r.cache.Do("stat:"+info.Version, func() any {
237 return cachedInfo{info, err}
238 })
239 }
240
241 if err := writeDiskStat(file, info); err != nil {
242 fmt.Fprintf(os.Stderr, "go: writing stat cache: %v\n", err)
243 }
244 }
245 return cachedInfo{info, err}
246 }).(cachedInfo)
247
248 if c.err != nil {
249 return nil, c.err
250 }
251 info := *c.info
252 return &info, nil
253 }
254
255 func (r *cachingRepo) Latest() (*RevInfo, error) {
256 c := r.cache.Do("latest:", func() any {
257 info, err := r.repo().Latest()
258
259
260 if err == nil {
261 r.cache.Do("stat:"+info.Version, func() any {
262 return cachedInfo{info, err}
263 })
264 if file, _, err := readDiskStat(r.path, info.Version); err != nil {
265 writeDiskStat(file, info)
266 }
267 }
268
269 return cachedInfo{info, err}
270 }).(cachedInfo)
271
272 if c.err != nil {
273 return nil, c.err
274 }
275 info := *c.info
276 return &info, nil
277 }
278
279 func (r *cachingRepo) GoMod(version string) ([]byte, error) {
280 type cached struct {
281 text []byte
282 err error
283 }
284 c := r.cache.Do("gomod:"+version, func() any {
285 file, text, err := readDiskGoMod(r.path, version)
286 if err == nil {
287
288 return cached{text, nil}
289 }
290
291 text, err = r.repo().GoMod(version)
292 if err == nil {
293 if err := checkGoMod(r.path, version, text); err != nil {
294 return cached{text, err}
295 }
296 if err := writeDiskGoMod(file, text); err != nil {
297 fmt.Fprintf(os.Stderr, "go: writing go.mod cache: %v\n", err)
298 }
299 }
300 return cached{text, err}
301 }).(cached)
302
303 if c.err != nil {
304 return nil, c.err
305 }
306 return append([]byte(nil), c.text...), nil
307 }
308
309 func (r *cachingRepo) Zip(dst io.Writer, version string) error {
310 return r.repo().Zip(dst, version)
311 }
312
313
314
315 func InfoFile(path, version string) (string, error) {
316 if !semver.IsValid(version) {
317 return "", fmt.Errorf("invalid version %q", version)
318 }
319
320 if file, _, err := readDiskStat(path, version); err == nil {
321 return file, nil
322 }
323
324 err := TryProxies(func(proxy string) error {
325 _, err := Lookup(proxy, path).Stat(version)
326 return err
327 })
328 if err != nil {
329 return "", err
330 }
331
332
333 file, err := CachePath(module.Version{Path: path, Version: version}, "info")
334 if err != nil {
335 return "", err
336 }
337 return file, nil
338 }
339
340
341
342
343 func GoMod(path, rev string) ([]byte, error) {
344
345
346 if !semver.IsValid(rev) {
347 if _, info, err := readDiskStat(path, rev); err == nil {
348 rev = info.Version
349 } else {
350 if errors.Is(err, statCacheErr) {
351 return nil, err
352 }
353 err := TryProxies(func(proxy string) error {
354 info, err := Lookup(proxy, path).Stat(rev)
355 if err == nil {
356 rev = info.Version
357 }
358 return err
359 })
360 if err != nil {
361 return nil, err
362 }
363 }
364 }
365
366 _, data, err := readDiskGoMod(path, rev)
367 if err == nil {
368 return data, nil
369 }
370
371 err = TryProxies(func(proxy string) (err error) {
372 data, err = Lookup(proxy, path).GoMod(rev)
373 return err
374 })
375 return data, err
376 }
377
378
379
380 func GoModFile(path, version string) (string, error) {
381 if !semver.IsValid(version) {
382 return "", fmt.Errorf("invalid version %q", version)
383 }
384 if _, err := GoMod(path, version); err != nil {
385 return "", err
386 }
387
388 file, err := CachePath(module.Version{Path: path, Version: version}, "mod")
389 if err != nil {
390 return "", err
391 }
392 return file, nil
393 }
394
395
396
397 func GoModSum(path, version string) (string, error) {
398 if !semver.IsValid(version) {
399 return "", fmt.Errorf("invalid version %q", version)
400 }
401 data, err := GoMod(path, version)
402 if err != nil {
403 return "", err
404 }
405 sum, err := goModSum(data)
406 if err != nil {
407 return "", err
408 }
409 return sum, nil
410 }
411
412 var errNotCached = fmt.Errorf("not in cache")
413
414
415
416
417
418 func readDiskStat(path, rev string) (file string, info *RevInfo, err error) {
419 file, data, err := readDiskCache(path, rev, "info")
420 if err != nil {
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440 if cfg.GOPROXY == "off" {
441 if file, info, err := readDiskStatByHash(path, rev); err == nil {
442 return file, info, nil
443 }
444 }
445 return file, nil, err
446 }
447 info = new(RevInfo)
448 if err := json.Unmarshal(data, info); err != nil {
449 return file, nil, errNotCached
450 }
451
452
453
454 data2, err := json.Marshal(info)
455 if err == nil && !bytes.Equal(data2, data) {
456 writeDiskCache(file, data)
457 }
458 return file, info, nil
459 }
460
461
462
463
464
465
466
467
468
469
470 func readDiskStatByHash(path, rev string) (file string, info *RevInfo, err error) {
471 if cfg.GOMODCACHE == "" {
472
473 return "", nil, errNotCached
474 }
475
476 if !codehost.AllHex(rev) || len(rev) < 12 {
477 return "", nil, errNotCached
478 }
479 rev = rev[:12]
480 cdir, err := cacheDir(path)
481 if err != nil {
482 return "", nil, errNotCached
483 }
484 dir, err := os.Open(cdir)
485 if err != nil {
486 return "", nil, errNotCached
487 }
488 names, err := dir.Readdirnames(-1)
489 dir.Close()
490 if err != nil {
491 return "", nil, errNotCached
492 }
493
494
495
496
497 var maxVersion string
498 suffix := "-" + rev + ".info"
499 err = errNotCached
500 for _, name := range names {
501 if strings.HasSuffix(name, suffix) {
502 v := strings.TrimSuffix(name, ".info")
503 if module.IsPseudoVersion(v) && semver.Compare(v, maxVersion) > 0 {
504 maxVersion = v
505 file, info, err = readDiskStat(path, strings.TrimSuffix(name, ".info"))
506 }
507 }
508 }
509 return file, info, err
510 }
511
512
513
514
515
516
517 var oldVgoPrefix = []byte("//vgo 0.0.")
518
519
520
521
522
523 func readDiskGoMod(path, rev string) (file string, data []byte, err error) {
524 file, data, err = readDiskCache(path, rev, "mod")
525
526
527 if bytes.HasPrefix(data, oldVgoPrefix) {
528 err = errNotCached
529 data = nil
530 }
531
532 if err == nil {
533 if err := checkGoMod(path, rev, data); err != nil {
534 return "", nil, err
535 }
536 }
537
538 return file, data, err
539 }
540
541
542
543
544
545
546 func readDiskCache(path, rev, suffix string) (file string, data []byte, err error) {
547 file, err = CachePath(module.Version{Path: path, Version: rev}, suffix)
548 if err != nil {
549 return "", nil, errNotCached
550 }
551 data, err = robustio.ReadFile(file)
552 if err != nil {
553 return file, nil, errNotCached
554 }
555 return file, data, nil
556 }
557
558
559
560 func writeDiskStat(file string, info *RevInfo) error {
561 if file == "" {
562 return nil
563 }
564 js, err := json.Marshal(info)
565 if err != nil {
566 return err
567 }
568 return writeDiskCache(file, js)
569 }
570
571
572
573 func writeDiskGoMod(file string, text []byte) error {
574 return writeDiskCache(file, text)
575 }
576
577
578
579 func writeDiskCache(file string, data []byte) error {
580 if file == "" {
581 return nil
582 }
583
584 if err := os.MkdirAll(filepath.Dir(file), 0777); err != nil {
585 return err
586 }
587
588
589
590 f, err := tempFile(filepath.Dir(file), filepath.Base(file), 0666)
591 if err != nil {
592 return err
593 }
594 defer func() {
595
596
597
598 if err != nil {
599 f.Close()
600 os.Remove(f.Name())
601 }
602 }()
603
604 if _, err := f.Write(data); err != nil {
605 return err
606 }
607 if err := f.Close(); err != nil {
608 return err
609 }
610 if err := robustio.Rename(f.Name(), file); err != nil {
611 return err
612 }
613
614 if strings.HasSuffix(file, ".mod") {
615 rewriteVersionList(filepath.Dir(file))
616 }
617 return nil
618 }
619
620
621 func tempFile(dir, prefix string, perm fs.FileMode) (f *os.File, err error) {
622 for i := 0; i < 10000; i++ {
623 name := filepath.Join(dir, prefix+strconv.Itoa(rand.Intn(1000000000))+".tmp")
624 f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm)
625 if os.IsExist(err) {
626 continue
627 }
628 break
629 }
630 return
631 }
632
633
634
635 func rewriteVersionList(dir string) (err error) {
636 if filepath.Base(dir) != "@v" {
637 base.Fatalf("go: internal error: misuse of rewriteVersionList")
638 }
639
640 listFile := filepath.Join(dir, "list")
641
642
643
644
645
646
647
648
649
650
651 f, err := lockedfile.Edit(listFile)
652 if err != nil {
653 return err
654 }
655 defer func() {
656 if cerr := f.Close(); cerr != nil && err == nil {
657 err = cerr
658 }
659 }()
660 infos, err := os.ReadDir(dir)
661 if err != nil {
662 return err
663 }
664 var list []string
665 for _, info := range infos {
666
667
668
669
670
671
672 name := info.Name()
673 if strings.HasSuffix(name, ".mod") {
674 v := strings.TrimSuffix(name, ".mod")
675 if v != "" && module.CanonicalVersion(v) == v {
676 list = append(list, v)
677 }
678 }
679 }
680 semver.Sort(list)
681
682 var buf bytes.Buffer
683 for _, v := range list {
684 buf.WriteString(v)
685 buf.WriteString("\n")
686 }
687 if fi, err := f.Stat(); err == nil && int(fi.Size()) == buf.Len() {
688 old := make([]byte, buf.Len()+1)
689 if n, err := f.ReadAt(old, 0); err == io.EOF && n == buf.Len() && bytes.Equal(buf.Bytes(), old) {
690 return nil
691 }
692 }
693
694
695 if err := f.Truncate(0); err != nil {
696 return err
697 }
698
699 if err := f.Truncate(int64(buf.Len())); err != nil {
700 return err
701 }
702
703
704 if _, err := f.Write(buf.Bytes()); err != nil {
705 f.Truncate(0)
706 return err
707 }
708
709 return nil
710 }
711
712 var (
713 statCacheOnce sync.Once
714 statCacheErr error
715 )
716
717
718
719 func checkCacheDir() error {
720 if cfg.GOMODCACHE == "" {
721
722
723 return fmt.Errorf("module cache not found: neither GOMODCACHE nor GOPATH is set")
724 }
725 if !filepath.IsAbs(cfg.GOMODCACHE) {
726 return fmt.Errorf("GOMODCACHE entry is relative; must be absolute path: %q.\n", cfg.GOMODCACHE)
727 }
728
729
730
731 statCacheOnce.Do(func() {
732 fi, err := os.Stat(cfg.GOMODCACHE)
733 if err != nil {
734 if !os.IsNotExist(err) {
735 statCacheErr = fmt.Errorf("could not create module cache: %w", err)
736 return
737 }
738 if err := os.MkdirAll(cfg.GOMODCACHE, 0777); err != nil {
739 statCacheErr = fmt.Errorf("could not create module cache: %w", err)
740 return
741 }
742 return
743 }
744 if !fi.IsDir() {
745 statCacheErr = fmt.Errorf("could not create module cache: %q is not a directory", cfg.GOMODCACHE)
746 return
747 }
748 })
749 return statCacheErr
750 }
751
View as plain text