diff --git a/Dockerfile b/Dockerfile index b8baef6..fb922bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # ------------------------------------------------------------------------ -from golang:1.12.1-alpine as build +from golang:1.12.3-alpine as build run apk add --update git env CGO_ENABLED 0 @@ -24,4 +24,7 @@ run apt-get update \ run yes |apt-get install -y grub2 grub-pc-bin grub-efi-amd64-bin \ && apt-get clean +run apt-get install -y ca-certificates \ + && apt-get clean + copy --from=build /go/bin/ /bin/ diff --git a/Dockerfile.store b/Dockerfile.store new file mode 100644 index 0000000..2ba363f --- /dev/null +++ b/Dockerfile.store @@ -0,0 +1,20 @@ +# ------------------------------------------------------------------------ +from golang:1.12.3-alpine as build +run apk add --update git + +env CGO_ENABLED 0 +arg GOPROXY + +workdir /src +add go.sum go.mod ./ +run go mod download + +add . ./ +run go test ./... +run go install ./cmd/dkl-store + +# ------------------------------------------------------------------------ +from alpine:3.9 +volume /srv/dkl-store +entrypoint ["/bin/dkl-store"] +copy --from=build /go/bin/ /bin/ diff --git a/cmd/dkl-local-server/upstream.go b/cmd/dkl-local-server/upstream.go index f319cd8..d05c5d2 100644 --- a/cmd/dkl-local-server/upstream.go +++ b/cmd/dkl-local-server/upstream.go @@ -1,6 +1,8 @@ package main import ( + "crypto/sha1" + "encoding/hex" "flag" "fmt" "io" @@ -40,29 +42,33 @@ func (ctx *renderContext) distFetch(path ...string) (outPath string, err error) return } + defer resp.Body.Close() + if resp.StatusCode != 200 { err = fmt.Errorf("wrong status: %s", resp.Status) - resp.Body.Close() return } - tempOutPath := filepath.Join(filepath.Dir(outPath), "._part_"+filepath.Base(outPath)) + fOut, err := os.Create(filepath.Join(filepath.Dir(outPath), "._part_"+filepath.Base(outPath))) + if err != nil { + return + } + + hash := sha1.New() + + out := io.MultiWriter(fOut, hash) done := make(chan error, 1) go func() { - defer resp.Body.Close() - defer close(done) + _, err = io.Copy(out, resp.Body) + fOut.Close() - out, err := os.Create(tempOutPath) if err != nil { - done <- err - return + os.Remove(fOut.Name()) } - defer out.Close() - - _, err = io.Copy(out, resp.Body) done <- err + close(done) }() wait: @@ -74,16 +80,24 @@ wait: case err = <-done: if err != nil { log.Print("fetch of ", subPath, " failed: ", err) - os.Remove(tempOutPath) return } - - log.Print("fetch of ", subPath, " finished") } - // TODO checksum + hexSum := hex.EncodeToString(hash.Sum(nil)) + log.Printf("fetch of %s finished (SHA1 checksum: %s)", subPath, hexSum) - err = os.Rename(tempOutPath, outPath) + if remoteSum := resp.Header.Get("X-Content-SHA1"); remoteSum != "" { + log.Printf("fetch of %s: remote SHA1 checksum: %s", subPath, remoteSum) + if remoteSum != hexSum { + err = fmt.Errorf("wrong SHA1 checksum: server=%s local=%s", remoteSum, hexSum) + log.Print("fetch of ", subPath, ": ", err) + os.Remove(fOut.Name()) + return + } + } + + err = os.Rename(fOut.Name(), outPath) return } diff --git a/cmd/dkl-store/main.go b/cmd/dkl-store/main.go new file mode 100644 index 0000000..f5c6785 --- /dev/null +++ b/cmd/dkl-store/main.go @@ -0,0 +1,134 @@ +package main + +import ( + "crypto/sha1" + "encoding/hex" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "path/filepath" + "time" +) + +var ( + bind = flag.String("bind", ":8080", "Bind address") + uploadToken = flag.String("upload-token", "", "Upload token (not uploads allowed if empty)") + storeDir = flag.String("store-dir", "/srv/dkl-store", "Store directory") +) + +func main() { + flag.Parse() + + http.HandleFunc("/", handleHTTP) + + log.Print("listening on ", *bind) + http.ListenAndServe(*bind, nil) +} + +func handleHTTP(w http.ResponseWriter, req *http.Request) { + filePath := filepath.Join(*storeDir, req.URL.Path) + + stat, err := os.Stat(filePath) + if err != nil { + writeErr(err, w) + return + } + + if stat.Mode().IsDir() { + http.NotFound(w, req) + return + } + + l := fmt.Sprintf("%s %s", req.Method, filePath) + log.Print(l) + defer log.Print(l, " done") + + switch req.Method { + case "GET": + sha1Hex, err := hashOf(filePath) + if err != nil { + writeErr(err, w) + return + } + + w.Header().Set("X-Content-SHA1", sha1Hex) + http.ServeFile(w, req, filePath) + + //case "POST": + // // TODO upload + + default: + http.NotFound(w, req) + return + } +} + +func writeErr(err error, w http.ResponseWriter) { + if os.IsNotExist(err) { + w.WriteHeader(http.StatusNotFound) + w.Write([]byte("Not found\n")) + return + } + + log.Print("internal error: ", err) + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("Internal error\n")) +} + +func hashOf(filePath string) (sha1Hex string, err error) { + sha1Path := filePath + ".sha1" + + fileStat, err := os.Stat(filePath) + if err != nil { + return + } + + sha1Stat, err := os.Stat(sha1Path) + + if err == nil { + if sha1Stat.ModTime().After(fileStat.ModTime()) { + // cached value is up-to-date + sha1HexBytes, readErr := ioutil.ReadFile(sha1Path) + + if readErr == nil { + sha1Hex = string(sha1HexBytes) + return + } + } + } else if !os.IsNotExist(err) { + // failed to stat cached value + return + } + + // no cached value could be read + log.Print("hashing ", filePath) + start := time.Now() + + // hash the input + f, err := os.Open(filePath) + if err != nil { + return + } + + defer f.Close() + + h := sha1.New() + _, err = io.Copy(h, f) + if err != nil { + return + } + + sha1Hex = hex.EncodeToString(h.Sum(nil)) + + log.Print("hashing ", filePath, " took ", time.Since(start).Truncate(time.Millisecond)) + + if writeErr := ioutil.WriteFile(sha1Path, []byte(sha1Hex), 0644); writeErr != nil { + log.Printf("WARNING: failed to cache SHA1: %v", writeErr) + } + + return +} diff --git a/modd.conf b/modd.conf index 5c48085..3dbcb67 100644 --- a/modd.conf +++ b/modd.conf @@ -1,6 +1,6 @@ **/*.go Dockerfile { prep: go test ./... prep: go install ./cmd/... - prep: docker build -t dls . + prep: docker build --build-arg GOPROXY=$GOPROXY -t dls . #daemon +sigterm: /var/lib/direktil/test-run }