direktil/local-server
4
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

move store out

Browse Source
This commit is contained in:
Mikaël Cluseau 2019-10-17 13:53:07 +11:00
parent 6ddc4d6da4
commit c0fc7bbe3d
3 changed files with 0 additions and 257 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Dockerfile.store
View File

@ -1,8 +0,0 @@
# ------------------------------------------------------------------------
from mcluseau/golang-builder:1.13.1 as build
# ------------------------------------------------------------------------
from alpine:3.10
volume /srv/dkl-store
entrypoint ["/bin/dkl-store"]
copy --from=build /go/bin/ /bin/

68
cmd/dkl-store-upload/main.go
View File

@ -1,68 +0,0 @@
package main
import (
"crypto/sha1"
"encoding/hex"
"flag"
"fmt"
"io"
"log"
"net/http"
"os"
)
var (
token = flag.String("token", "", "Upload token")
)
func main() {
flag.Parse()
args := flag.Args()
if len(args) != 2 {
fmt.Print("source file and target URL are required")
os.Exit(1)
}
inPath := args[0]
outURL := args[1]
in, err := os.Open(inPath)
fail(err)
// hash the file
log.Print("hashing...")
h := sha1.New()
_, err = io.Copy(h, in)
fail(err)
sha1Hex := hex.EncodeToString(h.Sum(nil))
log.Print("SHA1 of source: ", sha1Hex)
// rewind
_, err = in.Seek(0, os.SEEK_SET)
fail(err)
// upload
req, err := http.NewRequest("POST", outURL, in)
fail(err)
req.Header.Set("X-Content-SHA1", sha1Hex)
log.Print("uploading...")
resp, err := http.DefaultClient.Do(req)
fail(err)
if resp.StatusCode != http.StatusCreated {
log.Fatalf("unexpected HTTP status: %s", resp.Status)
}
log.Print("uploaded successfully")
}
func fail(err error) {
if err != nil {
log.Fatal(err)
}
}

181
cmd/dkl-store/main.go
View File

@ -1,181 +0,0 @@
package main
import (
"crypto/sha1"
"encoding/hex"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"time"
)
var (
bind = flag.String("bind", ":8080", "Bind address")
uploadToken = flag.String("upload-token", "", "Upload token (no uploads allowed if empty)")
storeDir = flag.String("store-dir", "/srv/dkl-store", "Store directory")
)
func main() {
log.SetFlags(log.LstdFlags | log.Lshortfile)
flag.Parse()
http.HandleFunc("/", handleHTTP)
log.Print("listening on ", *bind)
log.Fatal(http.ListenAndServe(*bind, nil))
}
func handleHTTP(w http.ResponseWriter, req *http.Request) {
filePath := filepath.Join(*storeDir, req.URL.Path)
l := fmt.Sprintf("%s %s", req.Method, filePath)
log.Print(l)
defer log.Print(l, " done")
stat, err := os.Stat(filePath)
if err != nil && !os.IsNotExist(err) {
writeErr(err, w)
return
} else if err == nil && stat.Mode().IsDir() {
http.NotFound(w, req)
return
}
switch req.Method {
case "GET", "HEAD":
sha1Hex, err := hashOf(filePath)
if err != nil {
writeErr(err, w)
return
}
w.Header().Set("X-Content-SHA1", sha1Hex)
http.ServeFile(w, req, filePath)
case "POST":
tmpOut := filepath.Join(filepath.Dir(filePath), "."+filepath.Base(filePath))
if err := os.MkdirAll(filepath.Dir(filePath), 0755); err != nil {
writeErr(err, w)
return
}
out, err := os.Create(tmpOut)
if err != nil {
writeErr(err, w)
return
}
h := sha1.New()
mw := io.MultiWriter(out, h)
_, err = io.Copy(mw, req.Body)
out.Close()
if err != nil {
os.Remove(tmpOut)
writeErr(err, w)
return
}
sha1Hex := hex.EncodeToString(h.Sum(nil))
log.Print("upload SHA1: ", sha1Hex)
reqSHA1 := req.Header.Get("X-Content-SHA1")
if reqSHA1 != "" {
if reqSHA1 != sha1Hex {
err = fmt.Errorf("upload SHA1 does not match given SHA1: %s", reqSHA1)
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(err.Error() + "\n"))
return
}
log.Print("upload SHA1 is as expected")
}
os.Rename(tmpOut, filePath)
if err := ioutil.WriteFile(filePath+".sha1", []byte(sha1Hex), 0644); err != nil {
writeErr(err, w)
return
}
w.WriteHeader(http.StatusCreated)
default:
http.NotFound(w, req)
return
}
}
func writeErr(err error, w http.ResponseWriter) {
if os.IsNotExist(err) {
w.WriteHeader(http.StatusNotFound)
w.Write([]byte("Not found\n"))
return
}
log.Output(2, fmt.Sprint("internal error: ", err))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Internal error\n"))
}
func hashOf(filePath string) (sha1Hex string, err error) {
sha1Path := filePath + ".sha1"
fileStat, err := os.Stat(filePath)
if err != nil {
return
}
sha1Stat, err := os.Stat(sha1Path)
if err == nil {
if sha1Stat.ModTime().After(fileStat.ModTime()) {
// cached value is up-to-date
sha1HexBytes, readErr := ioutil.ReadFile(sha1Path)
if readErr == nil {
sha1Hex = string(sha1HexBytes)
return
}
}
} else if !os.IsNotExist(err) {
// failed to stat cached value
return
}
// no cached value could be read
log.Print("hashing ", filePath)
start := time.Now()
// hash the input
f, err := os.Open(filePath)
if err != nil {
return
}
defer f.Close()
h := sha1.New()
_, err = io.Copy(h, f)
if err != nil {
return
}
sha1Hex = hex.EncodeToString(h.Sum(nil))
log.Print("hashing ", filePath, " took ", time.Since(start).Truncate(time.Millisecond))
if writeErr := ioutil.WriteFile(sha1Path, []byte(sha1Hex), 0644); writeErr != nil {
log.Printf("WARNING: failed to cache SHA1: %v", writeErr)
}
return
}