move store out
This commit is contained in:
Mikaël Cluseau
parent
6ddc4d6da4
commit
c0fc7bbe3d
|
|
@ -1,8 +0,0 @@
|
|||
# ------------------------------------------------------------------------
|
||||
from mcluseau/golang-builder:1.13.1 as build
|
||||
|
||||
# ------------------------------------------------------------------------
|
||||
from alpine:3.10
|
||||
volume /srv/dkl-store
|
||||
entrypoint ["/bin/dkl-store"]
|
||||
copy --from=build /go/bin/ /bin/
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
)
|
||||
|
||||
var (
|
||||
token = flag.String("token", "", "Upload token")
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
args := flag.Args()
|
||||
if len(args) != 2 {
|
||||
fmt.Print("source file and target URL are required")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
inPath := args[0]
|
||||
outURL := args[1]
|
||||
|
||||
in, err := os.Open(inPath)
|
||||
fail(err)
|
||||
|
||||
// hash the file
|
||||
log.Print("hashing...")
|
||||
h := sha1.New()
|
||||
_, err = io.Copy(h, in)
|
||||
fail(err)
|
||||
|
||||
sha1Hex := hex.EncodeToString(h.Sum(nil))
|
||||
|
||||
log.Print("SHA1 of source: ", sha1Hex)
|
||||
|
||||
// rewind
|
||||
_, err = in.Seek(0, os.SEEK_SET)
|
||||
fail(err)
|
||||
|
||||
// upload
|
||||
req, err := http.NewRequest("POST", outURL, in)
|
||||
fail(err)
|
||||
|
||||
req.Header.Set("X-Content-SHA1", sha1Hex)
|
||||
|
||||
log.Print("uploading...")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
fail(err)
|
||||
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
log.Fatalf("unexpected HTTP status: %s", resp.Status)
|
||||
}
|
||||
|
||||
log.Print("uploaded successfully")
|
||||
}
|
||||
|
||||
func fail(err error) {
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,181 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
bind = flag.String("bind", ":8080", "Bind address")
|
||||
uploadToken = flag.String("upload-token", "", "Upload token (no uploads allowed if empty)")
|
||||
storeDir = flag.String("store-dir", "/srv/dkl-store", "Store directory")
|
||||
)
|
||||
|
||||
func main() {
|
||||
log.SetFlags(log.LstdFlags | log.Lshortfile)
|
||||
flag.Parse()
|
||||
|
||||
http.HandleFunc("/", handleHTTP)
|
||||
|
||||
log.Print("listening on ", *bind)
|
||||
log.Fatal(http.ListenAndServe(*bind, nil))
|
||||
}
|
||||
|
||||
func handleHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
filePath := filepath.Join(*storeDir, req.URL.Path)
|
||||
|
||||
l := fmt.Sprintf("%s %s", req.Method, filePath)
|
||||
log.Print(l)
|
||||
defer log.Print(l, " done")
|
||||
|
||||
stat, err := os.Stat(filePath)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
writeErr(err, w)
|
||||
return
|
||||
} else if err == nil && stat.Mode().IsDir() {
|
||||
http.NotFound(w, req)
|
||||
return
|
||||
}
|
||||
|
||||
switch req.Method {
|
||||
case "GET", "HEAD":
|
||||
sha1Hex, err := hashOf(filePath)
|
||||
if err != nil {
|
||||
writeErr(err, w)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("X-Content-SHA1", sha1Hex)
|
||||
http.ServeFile(w, req, filePath)
|
||||
|
||||
case "POST":
|
||||
tmpOut := filepath.Join(filepath.Dir(filePath), "."+filepath.Base(filePath))
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(filePath), 0755); err != nil {
|
||||
writeErr(err, w)
|
||||
return
|
||||
}
|
||||
|
||||
out, err := os.Create(tmpOut)
|
||||
if err != nil {
|
||||
writeErr(err, w)
|
||||
return
|
||||
}
|
||||
|
||||
h := sha1.New()
|
||||
mw := io.MultiWriter(out, h)
|
||||
|
||||
_, err = io.Copy(mw, req.Body)
|
||||
out.Close()
|
||||
|
||||
if err != nil {
|
||||
os.Remove(tmpOut)
|
||||
|
||||
writeErr(err, w)
|
||||
return
|
||||
}
|
||||
|
||||
sha1Hex := hex.EncodeToString(h.Sum(nil))
|
||||
log.Print("upload SHA1: ", sha1Hex)
|
||||
|
||||
reqSHA1 := req.Header.Get("X-Content-SHA1")
|
||||
if reqSHA1 != "" {
|
||||
if reqSHA1 != sha1Hex {
|
||||
err = fmt.Errorf("upload SHA1 does not match given SHA1: %s", reqSHA1)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
w.Write([]byte(err.Error() + "\n"))
|
||||
return
|
||||
}
|
||||
|
||||
log.Print("upload SHA1 is as expected")
|
||||
}
|
||||
|
||||
os.Rename(tmpOut, filePath)
|
||||
|
||||
if err := ioutil.WriteFile(filePath+".sha1", []byte(sha1Hex), 0644); err != nil {
|
||||
writeErr(err, w)
|
||||
return
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
|
||||
default:
|
||||
http.NotFound(w, req)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func writeErr(err error, w http.ResponseWriter) {
|
||||
if os.IsNotExist(err) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
w.Write([]byte("Not found\n"))
|
||||
return
|
||||
}
|
||||
|
||||
log.Output(2, fmt.Sprint("internal error: ", err))
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
w.Write([]byte("Internal error\n"))
|
||||
}
|
||||
|
||||
func hashOf(filePath string) (sha1Hex string, err error) {
|
||||
sha1Path := filePath + ".sha1"
|
||||
|
||||
fileStat, err := os.Stat(filePath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
sha1Stat, err := os.Stat(sha1Path)
|
||||
|
||||
if err == nil {
|
||||
if sha1Stat.ModTime().After(fileStat.ModTime()) {
|
||||
// cached value is up-to-date
|
||||
sha1HexBytes, readErr := ioutil.ReadFile(sha1Path)
|
||||
|
||||
if readErr == nil {
|
||||
sha1Hex = string(sha1HexBytes)
|
||||
return
|
||||
}
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
// failed to stat cached value
|
||||
return
|
||||
}
|
||||
|
||||
// no cached value could be read
|
||||
log.Print("hashing ", filePath)
|
||||
start := time.Now()
|
||||
|
||||
// hash the input
|
||||
f, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
defer f.Close()
|
||||
|
||||
h := sha1.New()
|
||||
_, err = io.Copy(h, f)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
sha1Hex = hex.EncodeToString(h.Sum(nil))
|
||||
|
||||
log.Print("hashing ", filePath, " took ", time.Since(start).Truncate(time.Millisecond))
|
||||
|
||||
if writeErr := ioutil.WriteFile(sha1Path, []byte(sha1Hex), 0644); writeErr != nil {
|
||||
log.Printf("WARNING: failed to cache SHA1: %v", writeErr)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
Loading…
Reference in New Issue