dkl-store

This commit is contained in:
Mikaël Cluseau 2019-04-12 17:17:49 +01:00
parent 676c4bc21b
commit 6a0cd6da02
5 changed files with 188 additions and 17 deletions

View File

@ -1,5 +1,5 @@
# ------------------------------------------------------------------------ # ------------------------------------------------------------------------
from golang:1.12.1-alpine as build from golang:1.12.3-alpine as build
run apk add --update git run apk add --update git
env CGO_ENABLED 0 env CGO_ENABLED 0
@ -24,4 +24,7 @@ run apt-get update \
run yes |apt-get install -y grub2 grub-pc-bin grub-efi-amd64-bin \ run yes |apt-get install -y grub2 grub-pc-bin grub-efi-amd64-bin \
&& apt-get clean && apt-get clean
run apt-get install -y ca-certificates \
&& apt-get clean
copy --from=build /go/bin/ /bin/ copy --from=build /go/bin/ /bin/

20
Dockerfile.store Normal file
View File

@ -0,0 +1,20 @@
# ------------------------------------------------------------------------
from golang:1.12.3-alpine as build
run apk add --update git
env CGO_ENABLED 0
arg GOPROXY
workdir /src
add go.sum go.mod ./
run go mod download
add . ./
run go test ./...
run go install ./cmd/dkl-store
# ------------------------------------------------------------------------
from alpine:3.9
volume /srv/dkl-store
entrypoint ["/bin/dkl-store"]
copy --from=build /go/bin/ /bin/

View File

@ -1,6 +1,8 @@
package main package main
import ( import (
"crypto/sha1"
"encoding/hex"
"flag" "flag"
"fmt" "fmt"
"io" "io"
@ -40,29 +42,33 @@ func (ctx *renderContext) distFetch(path ...string) (outPath string, err error)
return return
} }
defer resp.Body.Close()
if resp.StatusCode != 200 { if resp.StatusCode != 200 {
err = fmt.Errorf("wrong status: %s", resp.Status) err = fmt.Errorf("wrong status: %s", resp.Status)
resp.Body.Close()
return return
} }
tempOutPath := filepath.Join(filepath.Dir(outPath), "._part_"+filepath.Base(outPath)) fOut, err := os.Create(filepath.Join(filepath.Dir(outPath), "._part_"+filepath.Base(outPath)))
if err != nil {
return
}
hash := sha1.New()
out := io.MultiWriter(fOut, hash)
done := make(chan error, 1) done := make(chan error, 1)
go func() { go func() {
defer resp.Body.Close() _, err = io.Copy(out, resp.Body)
defer close(done) fOut.Close()
out, err := os.Create(tempOutPath)
if err != nil { if err != nil {
done <- err os.Remove(fOut.Name())
return
} }
defer out.Close()
_, err = io.Copy(out, resp.Body)
done <- err done <- err
close(done)
}() }()
wait: wait:
@ -74,16 +80,24 @@ wait:
case err = <-done: case err = <-done:
if err != nil { if err != nil {
log.Print("fetch of ", subPath, " failed: ", err) log.Print("fetch of ", subPath, " failed: ", err)
os.Remove(tempOutPath)
return return
} }
log.Print("fetch of ", subPath, " finished")
} }
// TODO checksum hexSum := hex.EncodeToString(hash.Sum(nil))
log.Printf("fetch of %s finished (SHA1 checksum: %s)", subPath, hexSum)
err = os.Rename(tempOutPath, outPath) if remoteSum := resp.Header.Get("X-Content-SHA1"); remoteSum != "" {
log.Printf("fetch of %s: remote SHA1 checksum: %s", subPath, remoteSum)
if remoteSum != hexSum {
err = fmt.Errorf("wrong SHA1 checksum: server=%s local=%s", remoteSum, hexSum)
log.Print("fetch of ", subPath, ": ", err)
os.Remove(fOut.Name())
return
}
}
err = os.Rename(fOut.Name(), outPath)
return return
} }

134
cmd/dkl-store/main.go Normal file
View File

@ -0,0 +1,134 @@
package main
import (
"crypto/sha1"
"encoding/hex"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"time"
)
var (
bind = flag.String("bind", ":8080", "Bind address")
uploadToken = flag.String("upload-token", "", "Upload token (not uploads allowed if empty)")
storeDir = flag.String("store-dir", "/srv/dkl-store", "Store directory")
)
func main() {
flag.Parse()
http.HandleFunc("/", handleHTTP)
log.Print("listening on ", *bind)
http.ListenAndServe(*bind, nil)
}
func handleHTTP(w http.ResponseWriter, req *http.Request) {
filePath := filepath.Join(*storeDir, req.URL.Path)
stat, err := os.Stat(filePath)
if err != nil {
writeErr(err, w)
return
}
if stat.Mode().IsDir() {
http.NotFound(w, req)
return
}
l := fmt.Sprintf("%s %s", req.Method, filePath)
log.Print(l)
defer log.Print(l, " done")
switch req.Method {
case "GET":
sha1Hex, err := hashOf(filePath)
if err != nil {
writeErr(err, w)
return
}
w.Header().Set("X-Content-SHA1", sha1Hex)
http.ServeFile(w, req, filePath)
//case "POST":
// // TODO upload
default:
http.NotFound(w, req)
return
}
}
func writeErr(err error, w http.ResponseWriter) {
if os.IsNotExist(err) {
w.WriteHeader(http.StatusNotFound)
w.Write([]byte("Not found\n"))
return
}
log.Print("internal error: ", err)
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Internal error\n"))
}
func hashOf(filePath string) (sha1Hex string, err error) {
sha1Path := filePath + ".sha1"
fileStat, err := os.Stat(filePath)
if err != nil {
return
}
sha1Stat, err := os.Stat(sha1Path)
if err == nil {
if sha1Stat.ModTime().After(fileStat.ModTime()) {
// cached value is up-to-date
sha1HexBytes, readErr := ioutil.ReadFile(sha1Path)
if readErr == nil {
sha1Hex = string(sha1HexBytes)
return
}
}
} else if !os.IsNotExist(err) {
// failed to stat cached value
return
}
// no cached value could be read
log.Print("hashing ", filePath)
start := time.Now()
// hash the input
f, err := os.Open(filePath)
if err != nil {
return
}
defer f.Close()
h := sha1.New()
_, err = io.Copy(h, f)
if err != nil {
return
}
sha1Hex = hex.EncodeToString(h.Sum(nil))
log.Print("hashing ", filePath, " took ", time.Since(start).Truncate(time.Millisecond))
if writeErr := ioutil.WriteFile(sha1Path, []byte(sha1Hex), 0644); writeErr != nil {
log.Printf("WARNING: failed to cache SHA1: %v", writeErr)
}
return
}

View File

@ -1,6 +1,6 @@
**/*.go Dockerfile { **/*.go Dockerfile {
prep: go test ./... prep: go test ./...
prep: go install ./cmd/... prep: go install ./cmd/...
prep: docker build -t dls . prep: docker build --build-arg GOPROXY=$GOPROXY -t dls .
#daemon +sigterm: /var/lib/direktil/test-run #daemon +sigterm: /var/lib/direktil/test-run
} }