dkl-store

Dockerfile

@@ -1,5 +1,5 @@
 # ------------------------------------------------------------------------
-from golang:1.12.1-alpine as build
+from golang:1.12.3-alpine as build
 run apk add --update git
 
 env CGO_ENABLED 0
@@ -24,4 +24,7 @@ run apt-get update \
 run yes |apt-get install -y grub2 grub-pc-bin grub-efi-amd64-bin \
  && apt-get clean
 
+run apt-get install -y ca-certificates \
+ && apt-get clean
+
 copy --from=build /go/bin/ /bin/

Dockerfile.store

@@ -0,0 +1,20 @@
+# ------------------------------------------------------------------------
+from golang:1.12.3-alpine as build
+run apk add --update git
+
+env CGO_ENABLED 0
+arg GOPROXY
+
+workdir /src
+add go.sum go.mod ./
+run go mod download
+
+add . ./
+run go test ./...
+run go install ./cmd/dkl-store
+
+# ------------------------------------------------------------------------
+from alpine:3.9
+volume /srv/dkl-store
+entrypoint ["/bin/dkl-store"]
+copy --from=build /go/bin/ /bin/

cmd/dkl-local-server/upstream.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"crypto/sha1"
+	"encoding/hex"
 	"flag"
 	"fmt"
 	"io"
@@ -40,29 +42,33 @@ func (ctx *renderContext) distFetch(path ...string) (outPath string, err error)
 		return
 	}
 
+	defer resp.Body.Close()
+
 	if resp.StatusCode != 200 {
 		err = fmt.Errorf("wrong status: %s", resp.Status)
-		resp.Body.Close()
 		return
 	}
 
-	tempOutPath := filepath.Join(filepath.Dir(outPath), "._part_"+filepath.Base(outPath))
+	fOut, err := os.Create(filepath.Join(filepath.Dir(outPath), "._part_"+filepath.Base(outPath)))
+	if err != nil {
+		return
+	}
+
+	hash := sha1.New()
+
+	out := io.MultiWriter(fOut, hash)
 
 	done := make(chan error, 1)
 	go func() {
-		defer resp.Body.Close()
+		_, err = io.Copy(out, resp.Body)
-		defer close(done)
+		fOut.Close()
 
-		out, err := os.Create(tempOutPath)
 		if err != nil {
-			done <- err
+			os.Remove(fOut.Name())
-			return
 		}
 
-		defer out.Close()
-
-		_, err = io.Copy(out, resp.Body)
 		done <- err
+		close(done)
 	}()
 
 wait:
@@ -74,16 +80,24 @@ wait:
 	case err = <-done:
 		if err != nil {
 			log.Print("fetch of ", subPath, " failed: ", err)
-			os.Remove(tempOutPath)
 			return
 		}
-
-		log.Print("fetch of ", subPath, " finished")
 	}
 
-	// TODO checksum
+	hexSum := hex.EncodeToString(hash.Sum(nil))
+	log.Printf("fetch of %s finished (SHA1 checksum: %s)", subPath, hexSum)
 
-	err = os.Rename(tempOutPath, outPath)
+	if remoteSum := resp.Header.Get("X-Content-SHA1"); remoteSum != "" {
+		log.Printf("fetch of %s: remote SHA1 checksum: %s", subPath, remoteSum)
+		if remoteSum != hexSum {
+			err = fmt.Errorf("wrong SHA1 checksum: server=%s local=%s", remoteSum, hexSum)
+			log.Print("fetch of ", subPath, ": ", err)
+			os.Remove(fOut.Name())
+			return
+		}
+	}
+
+	err = os.Rename(fOut.Name(), outPath)
 
 	return
 }

cmd/dkl-store/main.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"crypto/sha1"
+	"encoding/hex"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+var (
+	bind        = flag.String("bind", ":8080", "Bind address")
+	uploadToken = flag.String("upload-token", "", "Upload token (not uploads allowed if empty)")
+	storeDir    = flag.String("store-dir", "/srv/dkl-store", "Store directory")
+)
+
+func main() {
+	flag.Parse()
+
+	http.HandleFunc("/", handleHTTP)
+
+	log.Print("listening on ", *bind)
+	http.ListenAndServe(*bind, nil)
+}
+
+func handleHTTP(w http.ResponseWriter, req *http.Request) {
+	filePath := filepath.Join(*storeDir, req.URL.Path)
+
+	stat, err := os.Stat(filePath)
+	if err != nil {
+		writeErr(err, w)
+		return
+	}
+
+	if stat.Mode().IsDir() {
+		http.NotFound(w, req)
+		return
+	}
+
+	l := fmt.Sprintf("%s %s", req.Method, filePath)
+	log.Print(l)
+	defer log.Print(l, " done")
+
+	switch req.Method {
+	case "GET":
+		sha1Hex, err := hashOf(filePath)
+		if err != nil {
+			writeErr(err, w)
+			return
+		}
+
+		w.Header().Set("X-Content-SHA1", sha1Hex)
+		http.ServeFile(w, req, filePath)
+
+	//case "POST":
+	//	// TODO upload
+
+	default:
+		http.NotFound(w, req)
+		return
+	}
+}
+
+func writeErr(err error, w http.ResponseWriter) {
+	if os.IsNotExist(err) {
+		w.WriteHeader(http.StatusNotFound)
+		w.Write([]byte("Not found\n"))
+		return
+	}
+
+	log.Print("internal error: ", err)
+	w.WriteHeader(http.StatusInternalServerError)
+	w.Write([]byte("Internal error\n"))
+}
+
+func hashOf(filePath string) (sha1Hex string, err error) {
+	sha1Path := filePath + ".sha1"
+
+	fileStat, err := os.Stat(filePath)
+	if err != nil {
+		return
+	}
+
+	sha1Stat, err := os.Stat(sha1Path)
+
+	if err == nil {
+		if sha1Stat.ModTime().After(fileStat.ModTime()) {
+			// cached value is up-to-date
+			sha1HexBytes, readErr := ioutil.ReadFile(sha1Path)
+
+			if readErr == nil {
+				sha1Hex = string(sha1HexBytes)
+				return
+			}
+		}
+	} else if !os.IsNotExist(err) {
+		// failed to stat cached value
+		return
+	}
+
+	// no cached value could be read
+	log.Print("hashing ", filePath)
+	start := time.Now()
+
+	// hash the input
+	f, err := os.Open(filePath)
+	if err != nil {
+		return
+	}
+
+	defer f.Close()
+
+	h := sha1.New()
+	_, err = io.Copy(h, f)
+	if err != nil {
+		return
+	}
+
+	sha1Hex = hex.EncodeToString(h.Sum(nil))
+
+	log.Print("hashing ", filePath, " took ", time.Since(start).Truncate(time.Millisecond))
+
+	if writeErr := ioutil.WriteFile(sha1Path, []byte(sha1Hex), 0644); writeErr != nil {
+		log.Printf("WARNING: failed to cache SHA1: %v", writeErr)
+	}
+
+	return
+}

modd.conf

@@ -1,6 +1,6 @@
 **/*.go Dockerfile {
   prep: go test ./...
   prep: go install ./cmd/...
-  prep: docker build -t dls .
+  prep: docker build --build-arg GOPROXY=$GOPROXY -t dls .
   #daemon +sigterm: /var/lib/direktil/test-run
 }