local-server/cmd/dkl-local-server/upstream.go

128 lines
2.5 KiB
Go
Raw Normal View History

2018-06-12 10:09:47 +00:00
package main
import (
2019-04-12 16:17:49 +00:00
"crypto/sha1"
"encoding/hex"
2018-06-12 10:09:47 +00:00
"flag"
2019-04-12 09:15:57 +00:00
"fmt"
2018-06-12 10:09:47 +00:00
"io"
"log"
"net/http"
"os"
gopath "path"
"path/filepath"
2019-12-19 17:34:18 +00:00
"strconv"
2018-06-12 10:09:47 +00:00
"time"
2019-12-19 17:34:18 +00:00
"github.com/dustin/go-humanize"
"github.com/miolini/datacounter"
2018-06-12 10:09:47 +00:00
)
var (
2023-03-22 19:43:27 +00:00
upstreamURL = flag.String("upstream", "https://dkl.novit.io/dist", "Upstream server for dist elements")
2018-06-12 10:09:47 +00:00
)
func (ctx *renderContext) distFetch(path ...string) (outPath string, err error) {
outPath = ctx.distFilePath(path...)
if _, err = os.Stat(outPath); err == nil {
return
} else if !os.IsNotExist(err) {
return
}
subPath := gopath.Join(path...)
log.Print("need to fetch ", subPath)
if err = os.MkdirAll(filepath.Dir(outPath), 0755); err != nil {
return
}
fullURL := *upstreamURL + "/" + subPath
resp, err := http.Get(fullURL)
if err != nil {
return
}
2019-04-12 16:17:49 +00:00
defer resp.Body.Close()
2019-04-12 09:15:57 +00:00
if resp.StatusCode != 200 {
err = fmt.Errorf("wrong status: %s", resp.Status)
return
}
2019-12-19 17:34:18 +00:00
length, _ := strconv.Atoi(resp.Header.Get("Content-Length"))
2019-04-12 16:17:49 +00:00
fOut, err := os.Create(filepath.Join(filepath.Dir(outPath), "._part_"+filepath.Base(outPath)))
if err != nil {
return
}
hash := sha1.New()
2019-12-19 17:34:18 +00:00
body := datacounter.NewReaderCounter(resp.Body)
2019-04-12 16:17:49 +00:00
out := io.MultiWriter(fOut, hash)
2018-06-12 10:09:47 +00:00
done := make(chan error, 1)
go func() {
2019-12-19 17:34:18 +00:00
_, err = io.Copy(out, body)
2019-04-12 16:17:49 +00:00
fOut.Close()
2018-06-12 10:09:47 +00:00
if err != nil {
2019-04-12 16:17:49 +00:00
os.Remove(fOut.Name())
2018-06-12 10:09:47 +00:00
}
done <- err
2019-04-12 16:17:49 +00:00
close(done)
2018-06-12 10:09:47 +00:00
}()
2019-12-19 17:34:18 +00:00
start := time.Now()
2018-06-12 10:09:47 +00:00
wait:
select {
case <-time.After(10 * time.Second):
2019-12-19 17:34:18 +00:00
status := ""
if length != 0 {
count := body.Count()
elapsedDuration := time.Since(start)
progress := float64(count) / float64(length)
elapsed := float64(elapsedDuration)
remaining := time.Duration(elapsed/progress - elapsed)
status = fmt.Sprintf(" (%.2f%%, ETA %v, %s/s)",
progress*100,
remaining.Truncate(time.Second),
humanize.Bytes(uint64(float64(count)/elapsedDuration.Seconds())))
}
log.Printf("still fetching %s%s...", subPath, status)
2018-06-12 10:09:47 +00:00
goto wait
case err = <-done:
if err != nil {
log.Print("fetch of ", subPath, " failed: ", err)
return
}
}
2019-04-12 16:17:49 +00:00
hexSum := hex.EncodeToString(hash.Sum(nil))
log.Printf("fetch of %s finished (SHA1 checksum: %s)", subPath, hexSum)
if remoteSum := resp.Header.Get("X-Content-SHA1"); remoteSum != "" {
log.Printf("fetch of %s: remote SHA1 checksum: %s", subPath, remoteSum)
if remoteSum != hexSum {
err = fmt.Errorf("wrong SHA1 checksum: server=%s local=%s", remoteSum, hexSum)
log.Print("fetch of ", subPath, ": ", err)
os.Remove(fOut.Name())
return
}
}
2019-04-12 09:22:39 +00:00
2019-04-12 16:17:49 +00:00
err = os.Rename(fOut.Name(), outPath)
2018-06-12 10:09:47 +00:00
return
}