Compare commits

...

107 Commits

Author SHA1 Message Date
f59eca6724 use GIT_TAG to override Version 2024-01-07 11:10:45 +01:00
b5b7272603 docker: update & use build cache 2024-01-07 10:56:42 +01:00
4f48866daa dir2config: templates: 'default' fn 2024-01-06 18:20:03 +01:00
b616b710cb hashes passwords support 2023-12-17 17:29:17 +01:00
c02f701c04 ssh: load more host key formats than rsa 2023-12-17 14:40:48 +01:00
7f429a863d clear initrd-v2 flag, use config 2023-11-27 15:46:58 +01:00
29ed01a19f hack: +install +docker-build 2023-11-27 13:52:44 +01:00
07e9dccd06 dir2config: add version 2023-11-25 16:47:20 +01:00
40d08139db public/unlock-store: idempotent call for passphrases
This allows the user to call it even after the store has been unlock in
order to get the admin token.
2023-11-09 08:59:30 +01:00
efa6193954 update Dockerfile to use hack/build 2023-11-07 15:17:47 +01:00
f7b708ce4b add server version/commit in logs and UI 2023-11-04 13:53:00 +01:00
41897c00b4 go mod update 2023-11-04 13:24:57 +01:00
ee5629643c named passphrases (+deletion by name) 2023-09-10 16:47:54 +02:00
34afe03818 cleanup old boot v1 code 2023-08-20 11:13:51 +02:00
25c2d20c19 ui: better ordering 2023-08-19 21:19:50 +02:00
c338522b33 allow store upload, + upload UIs for store and config 2023-08-19 21:17:23 +02:00
b6fa941fcc cleanup 2023-05-22 19:16:11 +02:00
7619998d8f dockerignore more 2023-05-22 19:05:33 +02:00
b6e7c55704 cleanup hosts ws 2023-05-18 19:55:52 +02:00
4ed50e3b78 upgrade dockerfile 2023-05-15 19:34:51 +02:00
dac6613646 fix code vs status in store errors 2023-05-15 19:27:36 +02:00
a8ccb6990b fix last refs to bootstrap-pods 2023-05-15 16:54:29 +02:00
b1cdb30622 boostrap pods -> static pods 2023-05-15 16:22:04 +02:00
50bb60823f yaml render func 2023-05-15 15:54:23 +02:00
482d3c83ba host_by_group: scope in cluster 2023-05-15 15:54:16 +02:00
74abbf9eda cluster: addons as list 2023-05-15 14:47:53 +02:00
76c1861017 bump deps 2023-05-01 16:33:56 +02:00
0d0494b825 dir2config: switch to includes instead of ad-hoc defaults 2023-05-01 16:09:54 +02:00
c6320049ff bump go 2023-04-16 22:54:19 +02:00
9e56acfc9a chore: simplify 2023-04-11 15:14:03 +02:00
6197369e04 go bump 2023-03-22 20:45:28 +01:00
d950bc6996 go mod tidy 2023-03-07 14:46:47 +01:00
18dc85d6fb missing notfound 2023-02-21 10:39:57 +01:00
26953cf703 secrets migration 2023-02-15 08:49:34 +01:00
1f03315897 log fix 2023-02-13 18:34:45 +01:00
5a6c0fa3d8 rework not found 2023-02-13 18:07:10 +01:00
4acdf88785 misc fixes 2023-02-13 17:31:37 +01:00
bde41c9859 host download tokens 2023-02-13 17:08:17 +01:00
1e3ac9a0fb store download & add key 2023-02-13 13:21:45 +01:00
1672b901d4 cosmestic 2023-02-12 18:59:14 +01:00
11f3c953e2 migration to new secrets nearly complete 2023-02-12 15:18:42 +01:00
3bc20e95cc secrets migration & restitution 2023-02-12 11:58:26 +01:00
1aefc5d2b7 go.mod: go 1.20 2023-02-09 08:58:48 +01:00
5c432e3b42 go mod update 2023-02-07 21:31:02 +01:00
b6c714fac7 downloads API, UI 2023-02-07 21:29:19 +01:00
e44303eab9 go mod update 2023-02-02 17:08:56 +01:00
2a9295e8e8 bump initrdv2 2023-02-02 00:32:38 +01:00
52ffbe9727 go mod update 2023-01-27 12:01:56 +01:00
811a3bddfd renew: don't use Renew, just create a new cert 2023-01-27 06:42:39 +01:00
227c341f6b renew: hande more error cases 2023-01-27 06:25:51 +01:00
153c37b591 secrets: more verbose errors 2023-01-27 06:13:05 +01:00
4ff85eaeb3 remove bootstrap pods from cluster, and render in hosts' context 2023-01-16 18:17:11 +01:00
76e02c6f31 allow configuration of grub-support version 2022-12-22 00:28:54 +01:00
93b32eb52a dockerfile: go 1.19.0 2022-08-05 16:06:36 +02:00
0fcd219268 boot-v2.iso: initial commit 2022-07-21 16:03:39 +02:00
18d3c42fc7 boot efi 2022-05-31 11:57:21 +02:00
645c617956 migrate boot.tar to initrd-v2 2022-05-31 09:52:43 +02:00
dacfc8c6ce upgrade deps 2022-04-28 10:01:21 +02:00
16a0ff0823 bootv2 support 2022-04-28 03:33:19 +02:00
0d298c9951 version dumps 2021-11-14 15:40:59 +01:00
3673a2f361 ssh acls preliminary support 2021-11-14 15:28:40 +01:00
4d92925170 env updates 2020-10-02 05:29:04 +02:00
a66d54d238 oops 2020-04-22 18:40:30 +02:00
748a028161 tls: automatic certificate renewal 2020-04-22 17:36:04 +02:00
5e667295ac small moves 2020-03-05 10:55:59 +01:00
a54d4bc15e cmdline query param for boot.iso 2020-03-05 00:06:49 +01:00
1ee5d1c15a fix: render template for addons and bootstrap pods 2019-12-26 11:10:23 +01:00
49a16fe550 boot.img: more verbose losetup run 2019-12-23 11:32:55 +01:00
984b2e0057 improve fetch status report 2019-12-19 18:34:18 +01:00
863a436915 don't write secret-data if not needed 2019-12-19 16:57:22 +01:00
55c72aefa8 sanity commit 2019-12-16 08:00:57 +01:00
8ce4e97922 fix: upload config without previous config + bump deps 2019-12-07 21:09:12 +01:00
f43f4fcec4 generate ssh secrets 2019-12-03 14:39:42 +01:00
6ef93489bd golang 1.13.4 2019-11-26 13:58:50 +01:00
85af5ccc36 Merge branch 'fix-api-ds' into 'master'
Fix API for bootstrap-pods daemonsets (apps/v1)

See merge request novit-nc/direktil/local-server!2
2019-11-08 04:42:41 +00:00
051b42fda8 Fix API for bootstrap-pods daemonsets (apps/v1) 2019-11-08 08:15:43 +11:00
39ea639cc3 dir2config: hosts_by_cluster 2019-10-24 14:06:18 +11:00
b5b8514c59 improve debugging 2019-10-23 11:03:35 +11:00
fec03e0a7e better merge of values 2019-10-19 15:08:41 +11:00
840824d438 Merge branch 'gfenollar' into 'master'
Add curl, govc and other stuff to ease ops

See merge request novit-nc/direktil/local-server!1
2019-10-18 07:18:14 +00:00
59fe6373dc more logs when loading defaults 2019-10-18 18:02:39 +11:00
43304de2ca Add double quotes for path with spaces 2019-10-18 18:00:04 +11:00
4ad32c64a6 Add curl, govc, and vmware upload script for quicker nodes updates 2019-10-18 16:52:45 +11:00
49c73be97a clear vendor 2019-10-17 14:21:08 +11:00
c0fc7bbe3d move store out 2019-10-17 13:53:07 +11:00
6ddc4d6da4 dir2config: bs pods: fix pod template 2019-10-17 01:07:27 +11:00
6c8835c5ab update dockerfile 2019-10-16 19:17:22 +11:00
4b0f5dca84 store: log when upload SHA1 is ok 2019-10-16 18:24:28 +11:00
48ab32f319 dkl-store-upload 2019-10-16 18:23:03 +11:00
a7158e9e56 store: upload support 2019-10-16 17:58:48 +11:00
daa919e953 template details rendering 2019-10-16 17:09:01 +11:00
50ee480caf dir2config: hosts_by_group returns maps 2019-10-16 16:46:04 +11:00
d27c4ed7a3 dir2config: more logs on cluster templates 2019-10-16 16:21:12 +11:00
36e1367522 store: prepare upload 2019-10-16 16:16:15 +11:00
4679da1c1e bootstrap pods support 2019-10-09 17:45:19 +11:00
dde0ad6975 ls 2019-10-09 16:58:28 +11:00
ee2779cc9d better map merge 2019-04-19 17:07:22 +01:00
a1fcd4093c so go-restful require different prefixes now 2019-04-18 18:47:31 +01:00
9b62d598bb factorize filter clauses 2019-04-15 18:56:31 +01:00
456722a616 dls: password support 2019-04-13 10:36:58 +01:00
6a0cd6da02 dkl-store 2019-04-12 17:17:49 +01:00
676c4bc21b todo 2019-04-12 10:22:39 +01:00
201bca587e check status code 2019-04-12 10:15:57 +01:00
024fcdd35c improve update-boot.sh 2019-03-29 19:51:23 +11:00
663b42ed47 update boot 2019-03-29 19:49:31 +11:00
c21b07572d docker & go updates 2019-03-19 14:57:57 +11:00
1391108d60 golang 1.12 2019-03-06 17:27:40 +11:00
1033 changed files with 21626 additions and 205094 deletions

2
.dockerignore Normal file
View File

@ -0,0 +1,2 @@
tmp
dist

6
.gitignore vendored
View File

@ -1,2 +1,8 @@
*.sw[po] *.sw[po]
modd-local.conf modd-local.conf
/tmp
/test-dir2config
/config.yaml
/dist
/go.work
/go.work.sum

View File

@ -1,23 +1,33 @@
# ------------------------------------------------------------------------ # ------------------------------------------------------------------------
from golang:1.11.5 as build from golang:1.21.5-bullseye as build
env pkg novit.nc/direktil/local-server run apt-get update && apt-get install -y git
copy vendor /go/src/${pkg}/vendor
copy pkg /go/src/${pkg}/pkg workdir /src
copy cmd /go/src/${pkg}/cmd
workdir /go/src/${pkg} copy go.mod go.sum ./
run go test ./... \ run \
&& go install ./cmd/... --mount=type=cache,id=gomod,target=/go/pkg/mod \
--mount=type=cache,id=gobuild,target=/root/.cache/go-build \
go mod download
arg GIT_TAG
copy . ./
run \
--mount=type=cache,id=gomod,target=/go/pkg/mod \
--mount=type=cache,id=gobuild,target=/root/.cache/go-build \
go test ./... && \
hack/build ./...
# ------------------------------------------------------------------------ # ------------------------------------------------------------------------
from debian:stretch from debian:bullseye
entrypoint ["/bin/dkl-local-server"] entrypoint ["/bin/dkl-local-server"]
env _uncache 1
run apt-get update \ run apt-get update \
&& apt-get install -y genisoimage gdisk dosfstools util-linux udev \ && yes |apt-get install -y genisoimage gdisk dosfstools util-linux udev binutils systemd \
grub2 grub-pc-bin grub-efi-amd64-bin ca-certificates curl openssh-client \
&& apt-get clean && apt-get clean
run yes |apt-get install -y grub2 grub-pc-bin grub-efi-amd64-bin \ copy --from=build /src/dist/ /bin/
&& apt-get clean
copy --from=build /go/bin/ /bin/

View File

@ -0,0 +1,138 @@
package main
import (
"bytes"
"fmt"
"io"
"log"
"os"
"strings"
"gopkg.in/yaml.v2"
)
func mergeIn(tgt, add map[any]any) {
mergeLoop:
for k, v := range add {
switch v := v.(type) {
case map[any]any:
if tgtV, ok := tgt[k]; ok {
switch tgtV := tgtV.(type) {
case map[any]any:
mergeIn(tgtV, v)
continue mergeLoop
}
}
}
tgt[k] = v
}
}
func assemble(path string) (yamlBytes []byte, err error) {
obj := map[any]any{}
if Debug {
log.Printf("assemble %q", path)
}
err = eachFragment(path, searchList, func(r io.Reader) (err error) {
m := map[any]any{}
err = yaml.NewDecoder(r).Decode(&m)
if err != nil {
return
}
mergeIn(obj, m)
return
})
if err != nil {
err = fmt.Errorf("failed to assemble %q: %w", path, err)
return
}
yamlBytes, err = yaml.Marshal(obj)
if err != nil {
return
}
if Debug {
log.Printf("assemble %q result:\n%s", path, yamlBytes)
}
return
}
func eachFragment(path string, searchList []FS, walk func(io.Reader) error) (err error) {
var r io.ReadCloser
for len(searchList) != 0 {
fs := searchList[0]
r, err = fs.Open(path + ".yaml")
if os.IsNotExist(err) {
searchList = searchList[1:]
continue
}
if err != nil {
return
}
// found and open
break
}
if r == nil {
err = fmt.Errorf("%s: %w", path, os.ErrNotExist)
return
}
ba, err := io.ReadAll(r)
r.Close()
if err != nil {
return
}
if Debug {
log.Print("fragment:\n", string(ba))
}
in := bytes.NewBuffer(ba)
for {
var line string
line, err = in.ReadString('\n')
if err == io.EOF {
break
}
if err != nil {
return
}
line = strings.TrimSpace(line)
if len(line) == 0 {
continue
}
includePath, found := strings.CutPrefix(line, "#!include ")
if !found {
continue // or break?
}
includePath = strings.TrimSpace(includePath)
if Debug {
log.Print("#!include ", includePath)
}
err = eachFragment(includePath, searchList, walk)
if err != nil {
err = fmt.Errorf("include %q: %w", includePath, err)
return
}
}
in = bytes.NewBuffer(ba)
err = walk(in)
return
}

34
cmd/dkl-dir2config/fs.go Normal file
View File

@ -0,0 +1,34 @@
package main
import (
"io"
iofs "io/fs"
)
type FS interface {
Open(path string) (io.ReadCloser, error)
List(path string) ([]string, error)
}
type fsFS struct{ iofs.FS }
func (fs fsFS) Open(path string) (io.ReadCloser, error) {
return fs.FS.Open(path)
}
func (fs fsFS) List(path string) (entries []string, err error) {
dirEnts, err := iofs.ReadDir(fs.FS, path)
if err != nil {
return
}
entries = make([]string, 0, len(dirEnts))
for _, ent := range dirEnts {
if ent.IsDir() {
continue
}
entries = append(entries, ent.Name())
}
return
}

38
cmd/dkl-dir2config/git.go Normal file
View File

@ -0,0 +1,38 @@
package main
import (
"io"
"sort"
"github.com/go-git/go-git/v5/plumbing/object"
)
type gitFS struct{ *object.Tree }
func (fs gitFS) Open(path string) (r io.ReadCloser, err error) {
f, err := fs.Tree.File(path)
if err != nil {
return
}
return f.Reader()
}
func (fs gitFS) List(path string) (entries []string, err error) {
tree, err := fs.Tree.Tree(path)
if err != nil {
return
}
entries = make([]string, 0, len(tree.Entries))
for _, ent := range tree.Entries {
if !ent.Mode.IsFile() {
continue
}
entries = append(entries, ent.Name)
}
sort.Strings(entries)
return
}

View File

@ -1,30 +1,42 @@
package main package main
import ( import (
"bytes"
"flag" "flag"
"fmt" "io/fs"
"log" "log"
"os" "os"
"path/filepath"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
yaml "gopkg.in/yaml.v2" yaml "gopkg.in/yaml.v2"
"novit.nc/direktil/pkg/localconfig"
"novit.nc/direktil/local-server/pkg/clustersconfig" "novit.tech/direktil/pkg/localconfig"
"novit.tech/direktil/local-server/pkg/clustersconfig"
) )
var Version = "dev"
var ( var (
dir = flag.String("in", ".", "Source directory") Debug = false
outPath = flag.String("out", "config.yaml", "Output file")
defaultsPath = flag.String("defaults", "defaults", "Path to the defaults") dir = flag.String("in", ".", "Source directory")
outPath = flag.String("out", "config.yaml", "Output file")
base fs.FS
src *clustersconfig.Config src *clustersconfig.Config
dst *localconfig.Config dst *localconfig.Config
) )
func init() {
flag.BoolVar(&Debug, "debug", Debug, "debug")
}
func loadSrc() { func loadSrc() {
var err error var err error
src, err = clustersconfig.FromDir(*dir, *defaultsPath) src, err = clustersconfig.FromDir(read, assemble, listBase, listMerged)
if err != nil { if err != nil {
log.Fatal("failed to load config from dir: ", err) log.Fatal("failed to load config from dir: ", err)
} }
@ -35,6 +47,16 @@ func main() {
log.SetFlags(log.Ltime | log.Lmicroseconds | log.Lshortfile) log.SetFlags(log.Ltime | log.Lmicroseconds | log.Lshortfile)
base = os.DirFS(*dir)
searchList = append(searchList, fsFS{base})
openIncludes()
if false {
assemble("hosts/m1")
log.Fatal("--- debug: end ---")
}
loadSrc() loadSrc()
dst = &localconfig.Config{ dst = &localconfig.Config{
@ -51,8 +73,6 @@ func main() {
// ---------------------------------------------------------------------- // ----------------------------------------------------------------------
for _, host := range src.Hosts { for _, host := range src.Hosts {
loadSrc() // FIXME ugly fix of some template caching or something
log.Print("rendering host ", host.Name) log.Print("rendering host ", host.Name)
ctx, err := newRenderContext(host, src) ctx, err := newRenderContext(host, src)
@ -71,23 +91,30 @@ func main() {
} }
ips = append(ips, host.IPs...) ips = append(ips, host.IPs...)
if ctx.Group.Versions["modules"] == "" { if ctx.Host.Versions["modules"] == "" {
// default modules' version to kernel's version // default modules' version to kernel's version
ctx.Group.Versions["modules"] = ctx.Group.Kernel ctx.Host.Versions["modules"] = ctx.Host.Kernel
} }
dst.Hosts = append(dst.Hosts, &localconfig.Host{ dst.Hosts = append(dst.Hosts, &localconfig.Host{
Name: host.Name, Name: host.Name,
ClusterName: ctx.Cluster.Name,
Labels: ctx.Labels,
Annotations: ctx.Annotations,
MACs: macs, MACs: macs,
IPs: ips, IPs: ips,
IPXE: ctx.Group.IPXE, // TODO render IPXE: ctx.Host.IPXE, // TODO render
Kernel: ctx.Group.Kernel, Kernel: ctx.Host.Kernel,
Initrd: ctx.Group.Initrd, Initrd: ctx.Host.Initrd,
Versions: ctx.Group.Versions, Versions: ctx.Host.Versions,
Config: ctx.Config(), BootstrapConfig: ctx.BootstrapConfig(),
Config: ctx.Config(),
}) })
} }
@ -99,39 +126,83 @@ func main() {
defer out.Close() defer out.Close()
out.Write([]byte("# dkl-dir2config " + Version + "\n"))
if err = yaml.NewEncoder(out).Encode(dst); err != nil { if err = yaml.NewEncoder(out).Encode(dst); err != nil {
log.Fatal("failed to render output: ", err) log.Fatal("failed to render output: ", err)
} }
} }
func renderAddons(cluster *clustersconfig.Cluster) string { func cfgPath(subPath string) string { return filepath.Join(*dir, subPath) }
if len(cluster.Addons) == 0 {
return "" func openIncludes() {
includesFile, err := base.Open("includes.yaml")
if os.IsNotExist(err) {
return
}
if err != nil {
log.Fatal("failed to open includes: ", err)
} }
addons := src.Addons[cluster.Addons] includes := make([]struct {
if addons == nil { Path string
log.Fatalf("cluster %q: no addons with name %q", cluster.Name, cluster.Addons) Branch string
Tag string
}, 0)
err = yaml.NewDecoder(includesFile).Decode(&includes)
if err != nil {
log.Fatal("failed to parse includes: ", err)
} }
clusterAsMap := asMap(cluster) for _, include := range includes {
clusterAsMap["kubernetes_svc_ip"] = cluster.KubernetesSvcIP().String() switch {
clusterAsMap["dns_svc_ip"] = cluster.DNSSvcIP().String() case include.Branch != "" || include.Tag != "":
p := cfgPath(include.Path) // FIXME parse git path to allow remote repos
buf := &bytes.Buffer{} var rev plumbing.Revision
for _, addon := range addons { switch {
fmt.Fprintf(buf, "---\n# addon: %s\n", addon.Name) case include.Branch != "":
err := addon.Execute(buf, clusterAsMap, nil) log.Printf("opening include path %q as git, branch %q", p, include.Branch)
rev = plumbing.Revision(plumbing.NewBranchReferenceName(include.Branch))
if err != nil { case include.Tag != "":
log.Fatalf("cluster %q: addons %q: failed to render %q: %v", log.Printf("opening include path %q as git, tag %q", p, include.Branch)
cluster.Name, cluster.Addons, addon.Name, err) rev = plumbing.Revision(plumbing.NewTagReferenceName(include.Branch))
}
repo, err := git.PlainOpen(p)
if err != nil {
log.Fatal("failed to open: ", err)
}
revH, err := repo.ResolveRevision(rev)
if err != nil {
log.Fatalf("failed to resolve revision %s: %v", rev, err)
}
log.Print(" -> resolved to commit ", *revH)
commit, err := repo.CommitObject(*revH)
if err != nil {
log.Fatal("failed to get commit object: ", err)
}
tree, err := commit.Tree()
if err != nil {
log.Fatal("failed to open git tree: ", err)
}
searchList = append(searchList, gitFS{tree})
default:
p := cfgPath(include.Path)
log.Printf("opening include path %q as raw dir", p)
searchList = append(searchList, fsFS{os.DirFS(p)})
} }
fmt.Fprintln(buf)
} }
return buf.String()
} }

View File

@ -0,0 +1,122 @@
package main
import (
"bytes"
"fmt"
"log"
"path"
"novit.tech/direktil/local-server/pkg/clustersconfig"
)
func clusterFuncs(clusterSpec *clustersconfig.Cluster) map[string]interface{} {
cluster := clusterSpec.Name
return map[string]interface{}{
"password": func(name, hash string) (s string) {
return fmt.Sprintf("{{ password %q %q %q | quote }}", cluster, name, hash)
},
"token": func(name string) (s string) {
return fmt.Sprintf("{{ token %q %q }}", cluster, name)
},
"ca_key": func(name string) (s string, err error) {
// TODO check CA exists
// ?ctx.clusterConfig.CA(name)
return fmt.Sprintf("{{ ca_key %q %q }}", cluster, name), nil
},
"ca_crt": func(name string) (s string, err error) {
// TODO check CA exists
return fmt.Sprintf("{{ ca_crt %q %q }}", cluster, name), nil
},
"ca_dir": func(name string) (s string, err error) {
return fmt.Sprintf("{{ ca_dir %q %q }}", cluster, name), nil
},
"hosts_by_cluster": func(cluster string) (hosts []interface{}) {
for _, host := range src.Hosts {
if host.Cluster == cluster {
hosts = append(hosts, asMap(host))
}
}
if len(hosts) == 0 {
log.Printf("WARNING: no hosts in cluster %q", cluster)
}
return
},
"hosts_by_group": func(group string) (hosts []interface{}) {
for _, host := range src.Hosts {
if host.Cluster == cluster && host.Group == group {
hosts = append(hosts, asMap(host))
}
}
if len(hosts) == 0 {
log.Printf("WARNING: no hosts in group %q", group)
}
return
},
}
}
func renderClusterTemplates(cluster *clustersconfig.Cluster, setName string,
templates []*clustersconfig.Template) []byte {
clusterAsMap := asMap(cluster)
clusterAsMap["kubernetes_svc_ip"] = cluster.KubernetesSvcIP().String()
clusterAsMap["dns_svc_ip"] = cluster.DNSSvcIP().String()
funcs := clusterFuncs(cluster)
log.Print("rendering cluster templates in ", setName)
buf := &bytes.Buffer{}
contextName := "cluster:" + cluster.Name
for _, t := range templates {
log.Print("- template: ", setName, ": ", t.Name)
fmt.Fprintf(buf, "---\n# %s: %s\n", setName, t.Name)
err := t.Execute(contextName, path.Join(setName, t.Name), buf, clusterAsMap, funcs)
if err != nil {
log.Fatalf("cluster %q: %s: failed to render %q: %v",
cluster.Name, setName, t.Name, err)
}
fmt.Fprintln(buf)
}
return buf.Bytes()
}
func renderAddons(cluster *clustersconfig.Cluster) string {
if len(cluster.Addons) == 0 {
return ""
}
buf := new(bytes.Buffer)
for _, addonSet := range cluster.Addons {
addons := src.Addons[addonSet]
if addons == nil {
log.Fatalf("cluster %q: no addons with name %q", cluster.Name, addonSet)
}
buf.Write(renderClusterTemplates(cluster, "addons", addons))
}
return buf.String()
}
type namePod struct {
Namespace string
Name string
Pod map[string]interface{}
}

View File

@ -2,21 +2,33 @@ package main
import ( import (
"bytes" "bytes"
"crypto/sha1"
"encoding/hex"
"fmt" "fmt"
"io"
"log" "log"
"path"
"reflect"
"strings"
yaml "gopkg.in/yaml.v2" yaml "gopkg.in/yaml.v2"
"novit.nc/direktil/local-server/pkg/clustersconfig" "novit.tech/direktil/pkg/config"
"novit.tech/direktil/local-server/pkg/clustersconfig"
) )
type renderContext struct { type renderContext struct {
Host *clustersconfig.Host Labels map[string]string
Group *clustersconfig.Group Annotations map[string]string
Cluster *clustersconfig.Cluster
Vars map[string]interface{} Host *clustersconfig.Host
ConfigTemplate *clustersconfig.Template Cluster *clustersconfig.Cluster
StaticPodsTemplate *clustersconfig.Template Vars map[string]any
BootstrapConfigTemplate *clustersconfig.Template
ConfigTemplate *clustersconfig.Template
StaticPodsTemplate *clustersconfig.Template
clusterConfig *clustersconfig.Config clusterConfig *clustersconfig.Config
} }
@ -28,95 +40,156 @@ func newRenderContext(host *clustersconfig.Host, cfg *clustersconfig.Config) (ct
return return
} }
group := cfg.Group(host.Group) vars := make(map[string]any)
if group == nil {
err = fmt.Errorf("no group named %q", host.Group)
return
}
vars := make(map[string]interface{}) for _, oVars := range []map[string]any{
for _, oVars := range []map[string]interface{}{
cluster.Vars, cluster.Vars,
group.Vars,
host.Vars, host.Vars,
} { } {
for k, v := range oVars { mapMerge(vars, oVars)
vars[k] = v
}
} }
return &renderContext{ return &renderContext{
Host: host, Labels: mergeLabels(cluster.Labels, host.Labels),
Group: group, Annotations: mergeLabels(cluster.Annotations, host.Annotations),
Cluster: cluster,
Vars: vars, Host: host,
ConfigTemplate: cfg.ConfigTemplate(group.Config), Cluster: cluster,
StaticPodsTemplate: cfg.StaticPodsTemplate(group.StaticPods), Vars: vars,
BootstrapConfigTemplate: cfg.ConfigTemplate(host.BootstrapConfig),
ConfigTemplate: cfg.ConfigTemplate(host.Config),
clusterConfig: cfg, clusterConfig: cfg,
}, nil }, nil
} }
func (ctx *renderContext) Config() string { func mergeLabels(sources ...map[string]string) map[string]string {
if ctx.ConfigTemplate == nil { ret := map[string]string{}
log.Fatalf("no such config: %q", ctx.Group.Config)
for _, src := range sources {
for k, v := range src {
ret[k] = v
}
} }
ctxMap := ctx.asMap() return ret
}
templateFuncs := ctx.templateFuncs(ctxMap) func mapMerge(target, source map[string]interface{}) {
for k, v := range source {
target[k] = genericMerge(target[k], v)
}
}
render := func(what string, t *clustersconfig.Template) (s string, err error) { func genericMerge(target, source interface{}) (result interface{}) {
buf := &bytes.Buffer{} srcV := reflect.ValueOf(source)
err = t.Execute(buf, ctxMap, templateFuncs) tgtV := reflect.ValueOf(target)
if err != nil {
log.Printf("host %s: failed to render %s [%q]: %v", ctx.Host.Name, what, t.Name, err) if srcV.Kind() == reflect.Map && tgtV.Kind() == reflect.Map {
return // XXX maybe more specific later
result = map[interface{}]interface{}{}
resultV := reflect.ValueOf(result)
tgtIt := tgtV.MapRange()
for tgtIt.Next() {
sv := srcV.MapIndex(tgtIt.Key())
if sv.Kind() == 0 {
resultV.SetMapIndex(tgtIt.Key(), tgtIt.Value())
continue
}
merged := genericMerge(tgtIt.Value().Interface(), sv.Interface())
resultV.SetMapIndex(tgtIt.Key(), reflect.ValueOf(merged))
}
srcIt := srcV.MapRange()
for srcIt.Next() {
if resultV.MapIndex(srcIt.Key()).Kind() != 0 {
continue // already done
}
resultV.SetMapIndex(srcIt.Key(), srcIt.Value())
} }
s = buf.String()
return return
} }
extraFuncs := ctx.templateFuncs(ctxMap) return source
}
extraFuncs["static_pods"] = func() (string, error) { func (ctx *renderContext) Name() string {
name := ctx.Group.StaticPods switch {
if len(name) == 0 { case ctx.Host != nil:
return "", fmt.Errorf("group %q has no static pods defined", ctx.Group.Name) return "host:" + ctx.Host.Name
} case ctx.Cluster != nil:
return "cluster:" + ctx.Cluster.Name
t := ctx.clusterConfig.StaticPodsTemplate(name) default:
if t == nil { return "unknown"
return "", fmt.Errorf("no static pods template named %q", name)
}
return render("static pods", t)
} }
}
buf := bytes.NewBuffer(make([]byte, 0, 4096)) func (ctx *renderContext) BootstrapConfig() string {
if err := ctx.ConfigTemplate.Execute(buf, ctxMap, extraFuncs); err != nil { if ctx.BootstrapConfigTemplate == nil {
log.Fatalf("failed to render config %q for host %q: %v", ctx.Group.Config, ctx.Host.Name, err) log.Fatalf("no such (bootstrap) config: %q", ctx.Host.BootstrapConfig)
} }
return ctx.renderConfig(ctx.BootstrapConfigTemplate)
}
func (ctx *renderContext) Config() string {
if ctx.ConfigTemplate == nil {
log.Fatalf("no such config: %q", ctx.Host.Config)
}
return ctx.renderConfig(ctx.ConfigTemplate)
}
func (ctx *renderContext) renderConfig(configTemplate *clustersconfig.Template) string {
buf := new(strings.Builder)
ctx.renderConfigTo(buf, configTemplate)
return buf.String() return buf.String()
} }
func (ctx *renderContext) StaticPods() (ba []byte, err error) { func (ctx *renderContext) renderConfigTo(buf io.Writer, configTemplate *clustersconfig.Template) {
if ctx.StaticPodsTemplate == nil { ctxName := ctx.Name()
log.Fatalf("no such static-pods: %q", ctx.Group.StaticPods)
}
ctxMap := ctx.asMap() ctxMap := ctx.asMap()
buf := bytes.NewBuffer(make([]byte, 0, 4096)) extraFuncs := ctx.templateFuncs(ctxMap)
if err = ctx.StaticPodsTemplate.Execute(buf, ctxMap, ctx.templateFuncs(ctxMap)); err != nil {
return extraFuncs["static_pods_files"] = func(dir string) (string, error) {
namePods := ctx.renderStaticPods()
defs := make([]config.FileDef, 0)
for _, namePod := range namePods {
name := namePod.Namespace + "_" + namePod.Name
ba, err := yaml.Marshal(namePod.Pod)
if err != nil {
return "", fmt.Errorf("static pod %s: failed to render: %v", name, err)
}
defs = append(defs, config.FileDef{
Path: path.Join(dir, name+".yaml"),
Mode: 0640,
Content: string(ba),
})
}
ba, err := yaml.Marshal(defs)
return string(ba), err
} }
ba = buf.Bytes() extraFuncs["machine_id"] = func() string {
return ba := sha1.Sum([]byte(ctx.Cluster.Name + "/" + ctx.Host.Name)) // TODO: check semantics of machine-id
return hex.EncodeToString(ba[:])
}
extraFuncs["version"] = func() string { return Version }
if err := configTemplate.Execute(ctxName, "config", buf, ctxMap, extraFuncs); err != nil {
log.Fatalf("failed to render config %q for host %q: %v", ctx.Host.Config, ctx.Host.Name, err)
}
} }
func (ctx *renderContext) templateFuncs(ctxMap map[string]interface{}) map[string]interface{} { func (ctx *renderContext) templateFuncs(ctxMap map[string]interface{}) map[string]interface{} {
@ -135,7 +208,7 @@ func (ctx *renderContext) templateFuncs(ctxMap map[string]interface{}) map[strin
} }
buf := &bytes.Buffer{} buf := &bytes.Buffer{}
err = req.Execute(buf, ctxMap, nil) err = req.Execute(ctx.Name(), "req:"+name, buf, ctxMap, nil)
if err != nil { if err != nil {
return return
} }
@ -159,26 +232,26 @@ func (ctx *renderContext) templateFuncs(ctxMap map[string]interface{}) map[strin
return return
} }
return map[string]interface{}{ funcs := clusterFuncs(ctx.Cluster)
"token": func(name string) (s string) { for k, v := range map[string]interface{}{
return fmt.Sprintf("{{ token %q %q }}", cluster, name) "default": func(value, defaultValue any) any {
}, switch v := value.(type) {
case string:
if v != "" {
return v
}
case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, float32, float64:
if v != 0 {
return v
}
"ca_key": func(name string) (s string, err error) { default:
// TODO check CA exists if v != nil {
// ?ctx.clusterConfig.CA(name) return v
return fmt.Sprintf("{{ ca_key %q %q }}", cluster, name), nil }
}
return defaultValue
}, },
"ca_crt": func(name string) (s string, err error) {
// TODO check CA exists
return fmt.Sprintf("{{ ca_crt %q %q }}", cluster, name), nil
},
"ca_dir": func(name string) (s string, err error) {
return fmt.Sprintf("{{ ca_dir %q %q }}", cluster, name), nil
},
"tls_key": func(name string) (string, error) { "tls_key": func(name string) (string, error) {
return getKeyCert(name, "tls_key") return getKeyCert(name, "tls_key")
}, },
@ -191,6 +264,14 @@ func (ctx *renderContext) templateFuncs(ctxMap map[string]interface{}) map[strin
return getKeyCert(name, "tls_dir") return getKeyCert(name, "tls_dir")
}, },
"ssh_host_keys": func(dir string) (s string) {
return fmt.Sprintf("{{ ssh_host_keys %q %q \"\"}}",
dir, cluster)
},
"host_download_token": func() (s string) {
return "{{ host_download_token }}"
},
"hosts_of_group": func() (hosts []interface{}) { "hosts_of_group": func() (hosts []interface{}) {
hosts = make([]interface{}, 0) hosts = make([]interface{}, 0)
@ -213,7 +294,10 @@ func (ctx *renderContext) templateFuncs(ctxMap map[string]interface{}) map[strin
} }
return return
}, },
} {
funcs[k] = v
} }
return funcs
} }
func (ctx *renderContext) asMap() map[string]interface{} { func (ctx *renderContext) asMap() map[string]interface{} {

View File

@ -0,0 +1,54 @@
package main
import (
"testing"
yaml "gopkg.in/yaml.v2"
)
func TestMerge(t *testing.T) {
if v := genericMerge("a", "b"); v != "b" {
t.Errorf("got %q", v)
}
if v := unparse(genericMerge(parse(`
a: t
b: t
m:
a1: t
b1: t
`), parse(`
a: s
c: s
m:
a1: s
c1: s
`))); "\n"+v != `
a: s
b: t
c: s
m:
a1: s
b1: t
c1: s
` {
t.Errorf("got\n%s", v)
}
}
func parse(s string) (r interface{}) {
r = map[string]interface{}{}
err := yaml.Unmarshal([]byte(s), r)
if err != nil {
panic(err)
}
return
}
func unparse(v interface{}) (s string) {
ba, err := yaml.Marshal(v)
if err != nil {
panic(err)
}
return string(ba)
}

View File

@ -0,0 +1,77 @@
package main
import (
"bytes"
"fmt"
"io"
"log"
yaml "gopkg.in/yaml.v2"
"novit.tech/direktil/local-server/pkg/clustersconfig"
)
func (ctx *renderContext) renderStaticPods() (pods []namePod) {
if ctx.Host.StaticPods == "" {
return
}
staticPods, ok := src.StaticPods[ctx.Host.StaticPods]
if !ok {
log.Fatalf("no static pods template named %q", ctx.Host.StaticPods)
}
// render static pods
parts := bytes.Split(ctx.renderHostTemplates("static-pods", staticPods), []byte("\n---\n"))
for _, part := range parts {
buf := bytes.NewBuffer(part)
dec := yaml.NewDecoder(buf)
for n := 0; ; n++ {
str := buf.String()
podMap := map[string]interface{}{}
err := dec.Decode(podMap)
if err == io.EOF {
break
} else if err != nil {
log.Fatalf("static pod %d: failed to parse: %v\n%s", n, err, str)
}
if len(podMap) == 0 {
continue
}
if podMap["metadata"] == nil {
log.Fatalf("static pod %d: no metadata\n%s", n, buf.String())
}
md := podMap["metadata"].(map[interface{}]interface{})
namespace := md["namespace"].(string)
name := md["name"].(string)
pods = append(pods, namePod{namespace, name, podMap})
}
}
return
}
func (ctx *renderContext) renderHostTemplates(setName string,
templates []*clustersconfig.Template) []byte {
log.Print("rendering host templates in ", setName)
buf := &bytes.Buffer{}
for _, t := range templates {
log.Print("- template: ", setName, ": ", t.Name)
fmt.Fprintf(buf, "---\n# %s: %s\n", setName, t.Name)
ctx.renderConfigTo(buf, t)
fmt.Fprintln(buf)
}
return buf.Bytes()
}

View File

@ -0,0 +1,60 @@
package main
import (
"fmt"
"io"
"io/ioutil"
"os"
"sort"
)
var searchList = make([]FS, 0)
// read the first file matching path in the search list
func read(path string) (ba []byte, err error) {
for _, fs := range searchList {
var r io.ReadCloser
r, err = fs.Open(path)
if os.IsNotExist(err) {
continue
}
if err != nil {
return
}
defer r.Close()
return ioutil.ReadAll(r)
}
err = fmt.Errorf("%s: %w", path, os.ErrNotExist)
return
}
func listBase(path string) ([]string, error) {
return fsFS{base}.List(path)
}
func listMerged(path string) (entries []string, err error) {
seen := map[string]bool{}
for _, fs := range searchList {
var fsEnts []string
fsEnts, err = fs.List(path)
if os.IsNotExist(err) {
err = nil
continue
}
if err != nil {
return
}
for _, ent := range fsEnts {
if !seen[ent] {
entries = append(entries, ent)
seen[ent] = true
}
}
}
sort.Strings(entries)
return
}

View File

@ -1,22 +1,14 @@
package main package main
import ( import (
"flag"
"log" "log"
"net/http" "net/http"
) )
var ( var adminToken string
hostsToken = flag.String("hosts-token", "", "Token to give to access /hosts (open is none)")
adminToken = flag.String("admin-token", "", "Token to give to access to admin actions (open is none)")
)
func authorizeHosts(r *http.Request) bool {
return authorizeToken(r, *hostsToken)
}
func authorizeAdmin(r *http.Request) bool { func authorizeAdmin(r *http.Request) bool {
return authorizeToken(r, *adminToken) return authorizeToken(r, adminToken)
} }
func authorizeToken(r *http.Request, token string) bool { func authorizeToken(r *http.Request, token string) bool {
@ -26,11 +18,28 @@ func authorizeToken(r *http.Request, token string) bool {
} }
reqToken := r.Header.Get("Authorization") reqToken := r.Header.Get("Authorization")
if reqToken != "" {
return reqToken == "Bearer "+token
}
return reqToken == "Bearer "+token return r.URL.Query().Get("token") == token
} }
func forbidden(w http.ResponseWriter, r *http.Request) { func forbidden(w http.ResponseWriter, r *http.Request) {
log.Printf("denied access to %s from %s", r.RequestURI, r.RemoteAddr) log.Printf("denied access to %s from %s", r.URL.Path, r.RemoteAddr)
http.Error(w, "Forbidden", http.StatusForbidden) http.Error(w, "Forbidden", http.StatusForbidden)
} }
func requireToken(token string, handler http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if !authorizeToken(req, token) {
forbidden(w, req)
return
}
handler.ServeHTTP(w, req)
})
}
func requireAdmin(handler http.Handler) http.Handler {
return requireToken(adminToken, handler)
}

View File

@ -3,6 +3,7 @@ package main
import ( import (
"archive/tar" "archive/tar"
"compress/gzip" "compress/gzip"
"flag"
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
@ -56,8 +57,10 @@ func buildBootImgGZ(out io.Writer, ctx *renderContext) (err error) {
return return
} }
var grubSupportVersion = flag.String("grub-support", "1.0.1", "GRUB support version")
func setupBootImage(bootImg *os.File, ctx *renderContext) (err error) { func setupBootImage(bootImg *os.File, ctx *renderContext) (err error) {
path, err := ctx.distFetch("grub-support", "1.0.0") path, err := ctx.distFetch("grub-support", *grubSupportVersion)
if err != nil { if err != nil {
return return
} }
@ -81,7 +84,10 @@ func setupBootImage(bootImg *os.File, ctx *renderContext) (err error) {
return return
} }
devb, err := exec.Command("losetup", "--find", "--show", "--partscan", bootImg.Name()).CombinedOutput() log.Print("running losetup...")
cmd := exec.Command("losetup", "--find", "--show", "--partscan", bootImg.Name())
cmd.Stderr = os.Stderr
devb, err := cmd.Output()
if err != nil { if err != nil {
return return
} }

View File

@ -8,28 +8,25 @@ import (
"os" "os"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"strconv"
"github.com/cespare/xxhash"
) )
func buildBootISO(out io.Writer, ctx *renderContext) error { func buildBootISO(out io.Writer, ctx *renderContext) (err error) {
tempDir, err := ioutil.TempDir("/tmp", "iso-") tempDir, err := ioutil.TempDir("/tmp", "iso-v2-")
if err != nil { if err != nil {
return err return
} }
defer os.RemoveAll(tempDir) defer os.RemoveAll(tempDir)
cp := func(src, dst string) error { buildRes := func(build func(out io.Writer, ctx *renderContext) error, dst string) (err error) {
log.Printf("iso: adding %s as %s", src, dst) log.Printf("iso-v2: building %s", dst)
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
outPath := filepath.Join(tempDir, dst) outPath := filepath.Join(tempDir, dst)
if err := os.MkdirAll(filepath.Dir(outPath), 0755); err != nil { if err = os.MkdirAll(filepath.Dir(outPath), 0755); err != nil {
return err return err
} }
@ -37,32 +34,55 @@ func buildBootISO(out io.Writer, ctx *renderContext) error {
if err != nil { if err != nil {
return err return err
} }
defer out.Close() defer out.Close()
_, err = io.Copy(out, in) err = build(out, ctx)
if err != nil {
return
}
return err return err
} }
err = func() error { err = func() (err error) {
// grub // grub
if err := os.MkdirAll(filepath.Join(tempDir, "grub"), 0755); err != nil { if err = os.MkdirAll(filepath.Join(tempDir, "grub"), 0755); err != nil {
return err return
} }
// create a tag file
bootstrapBytes, _, err := ctx.BootstrapConfig()
if err != nil {
return
}
h := xxhash.New()
fmt.Fprintln(h, ctx.Host.Kernel)
h.Write(bootstrapBytes)
tag := "dkl-" + strconv.FormatUint(h.Sum64(), 32) + ".tag"
f, err := os.Create(filepath.Join(tempDir, tag))
if err != nil {
return
}
f.Write([]byte("direktil marker file\n"))
f.Close()
err = ioutil.WriteFile(filepath.Join(tempDir, "grub", "grub.cfg"), []byte(` err = ioutil.WriteFile(filepath.Join(tempDir, "grub", "grub.cfg"), []byte(`
search --set=root --file /config.yaml search --set=root --file /`+tag+`
insmod all_video insmod all_video
set timeout=3 set timeout=3
menuentry "Direktil" { menuentry "Direktil" {
linux /vmlinuz direktil.boot=DEVNAME=sr0 direktil.boot.fs=iso9660 linux /vmlinuz `+ctx.CmdLine+`
initrd /initrd initrd /initrd
} }
`), 0644) `), 0644)
if err != nil { if err != nil {
return err return
} }
coreImgPath := filepath.Join(tempDir, "grub", "core.img") coreImgPath := filepath.Join(tempDir, "grub", "core.img")
@ -117,50 +137,9 @@ menuentry "Direktil" {
return err return err
} }
// config
cfgBytes, cfg, err := ctx.Config()
if err != nil {
return err
}
ioutil.WriteFile(filepath.Join(tempDir, "config.yaml"), cfgBytes, 0600)
// kernel and initrd // kernel and initrd
type distCopy struct { buildRes(fetchKernel, "vmlinuz")
Src []string buildRes(buildInitrd, "initrd")
Dst string
}
copies := []distCopy{
{Src: []string{"kernels", ctx.Host.Kernel}, Dst: "vmlinuz"},
{Src: []string{"initrd", ctx.Host.Initrd}, Dst: "initrd"},
}
// layers
for _, layer := range cfg.Layers {
layerVersion := ctx.Host.Versions[layer]
if layerVersion == "" {
return fmt.Errorf("layer %q not mapped to a version", layer)
}
copies = append(copies,
distCopy{
Src: []string{"layers", layer, layerVersion},
Dst: filepath.Join("current", "layers", layer+".fs"),
})
}
for _, copy := range copies {
outPath, err := ctx.distFetch(copy.Src...)
if err != nil {
return err
}
err = cp(outPath, copy.Dst)
if err != nil {
return err
}
}
// build the ISO // build the ISO
mkisofs, err := exec.LookPath("genisoimage") mkisofs, err := exec.LookPath("genisoimage")

View File

@ -2,11 +2,13 @@ package main
import ( import (
"archive/tar" "archive/tar"
"fmt" "bytes"
"io" "io"
"io/ioutil"
"log" "log"
"os" "os"
"path/filepath"
"novit.tech/direktil/local-server/pkg/utf16"
) )
func rmTempFile(f *os.File) { func rmTempFile(f *os.File) {
@ -21,7 +23,7 @@ func buildBootTar(out io.Writer, ctx *renderContext) (err error) {
defer arch.Close() defer arch.Close()
archAdd := func(path string, ba []byte) (err error) { archAdd := func(path string, ba []byte) (err error) {
err = arch.WriteHeader(&tar.Header{Name: path, Size: int64(len(ba))}) err = arch.WriteHeader(&tar.Header{Name: path, Mode: 0640, Size: int64(len(ba))})
if err != nil { if err != nil {
return return
} }
@ -29,70 +31,95 @@ func buildBootTar(out io.Writer, ctx *renderContext) (err error) {
return return
} }
// config // kernel
cfgBytes, cfg, err := ctx.Config() kernelPath, err := ctx.distFetch("kernels", ctx.Host.Kernel)
if err != nil { if err != nil {
return err return
} }
archAdd("config.yaml", cfgBytes) kernelBytes, err := ioutil.ReadFile(kernelPath)
if err != nil {
// add "current" elements return
type distCopy struct {
Src []string
Dst string
} }
// kernel and initrd err = archAdd("current/vmlinuz", kernelBytes)
copies := []distCopy{ if err != nil {
{Src: []string{"kernels", ctx.Host.Kernel}, Dst: "current/vmlinuz"}, return
{Src: []string{"initrd", ctx.Host.Initrd}, Dst: "current/initrd"},
} }
// layers // initrd
for _, layer := range cfg.Layers { initrd := new(bytes.Buffer)
layerVersion := ctx.Host.Versions[layer] err = buildInitrd(initrd, ctx)
if layerVersion == "" { if err != nil {
return fmt.Errorf("layer %q not mapped to a version", layer) return
}
copies = append(copies,
distCopy{
Src: []string{"layers", layer, layerVersion},
Dst: filepath.Join("current", "layers", layer+".fs"),
})
} }
for _, copy := range copies { err = archAdd("current/initrd", initrd.Bytes())
outPath, err := ctx.distFetch(copy.Src...) if err != nil {
if err != nil { return
return err
}
f, err := os.Open(outPath)
if err != nil {
return err
}
defer f.Close()
stat, err := f.Stat()
if err != nil {
return err
}
if err = arch.WriteHeader(&tar.Header{
Name: copy.Dst,
Size: stat.Size(),
}); err != nil {
return err
}
_, err = io.Copy(arch, f)
if err != nil {
return err
}
} }
// done
return nil
}
func buildBootEFITar(out io.Writer, ctx *renderContext) (err error) {
arch := tar.NewWriter(out)
defer arch.Close()
archAdd := func(path string, ba []byte) (err error) {
err = arch.WriteHeader(&tar.Header{Name: path, Mode: 0640, Size: int64(len(ba))})
if err != nil {
return
}
_, err = arch.Write(ba)
return
}
const (
prefix = "EFI/dkl/"
efiPrefix = "\\EFI\\dkl\\"
)
// boot.csv
// -> annoyingly it's UTF-16...
bootCsvBytes := utf16.FromUTF8([]byte("" +
"current_kernel.efi,dkl current,initrd=" + efiPrefix + "current_initrd.img,Direktil current\n" +
"previous_kernel.efi,dkl previous,initrd=" + efiPrefix + "previous_initrd.img,Direktil previous\n"))
err = archAdd(prefix+"BOOT.CSV", []byte(bootCsvBytes))
if err != nil {
return
}
// kernel
kernelPath, err := ctx.distFetch("kernels", ctx.Host.Kernel)
if err != nil {
return
}
kernelBytes, err := ioutil.ReadFile(kernelPath)
if err != nil {
return
}
err = archAdd(prefix+"current_kernel.efi", kernelBytes)
if err != nil {
return
}
// initrd
initrd := new(bytes.Buffer)
err = buildInitrd(initrd, ctx)
if err != nil {
return
}
err = archAdd(prefix+"current_initrd.img", initrd.Bytes())
if err != nil {
return
}
// done
return nil return nil
} }

View File

@ -0,0 +1,142 @@
package main
import (
"archive/tar"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"os"
yaml "gopkg.in/yaml.v2"
"novit.tech/direktil/pkg/cpiocat"
)
func renderBootstrapConfig(w http.ResponseWriter, r *http.Request, ctx *renderContext, asJson bool) (err error) {
log.Printf("sending bootstrap config for %q", ctx.Host.Name)
_, cfg, err := ctx.BootstrapConfig()
if err != nil {
return err
}
if asJson {
err = json.NewEncoder(w).Encode(cfg)
} else {
err = yaml.NewEncoder(w).Encode(cfg)
}
return nil
}
func buildInitrd(out io.Writer, ctx *renderContext) (err error) {
_, cfg, err := ctx.Config()
if err != nil {
return
}
cat := cpiocat.New(out)
// initrd
initrdPath, err := ctx.distFetch("initrd", ctx.Host.Initrd)
if err != nil {
return
}
cat.AppendArchFile(initrdPath)
// embedded layers (modules)
for _, layer := range cfg.Layers {
switch layer {
case "modules":
layerVersion := ctx.Host.Versions[layer]
modulesPath, err := ctx.distFetch("layers", layer, layerVersion)
if err != nil {
return err
}
cat.AppendFile(modulesPath, "modules.sqfs")
}
}
// config
cfgBytes, _, err := ctx.BootstrapConfig()
if err != nil {
return
}
cat.AppendBytes(cfgBytes, "config.yaml", 0600)
// ssh keys
// FIXME we want a bootstrap-stage key instead of the real host key
for _, format := range []string{"rsa", "dsa", "ecdsa", "ed25519"} {
cat.AppendBytes(cfg.FileContent("/etc/ssh/ssh_host_"+format+"_key"), "id_"+format, 0600)
}
return cat.Close()
}
func buildBootstrap(out io.Writer, ctx *renderContext) (err error) {
arch := tar.NewWriter(out)
defer arch.Close()
// config
cfgBytes, cfg, err := ctx.Config()
if err != nil {
return err
}
err = arch.WriteHeader(&tar.Header{Name: "config.yaml", Size: int64(len(cfgBytes))})
if err != nil {
return
}
_, err = arch.Write(cfgBytes)
if err != nil {
return
}
// layers
for _, layer := range cfg.Layers {
if layer == "modules" {
continue // modules are with the kernel in boot v2
}
layerVersion := ctx.Host.Versions[layer]
if layerVersion == "" {
return fmt.Errorf("layer %q not mapped to a version", layer)
}
outPath, err := ctx.distFetch("layers", layer, layerVersion)
if err != nil {
return err
}
f, err := os.Open(outPath)
if err != nil {
return err
}
defer f.Close()
stat, err := f.Stat()
if err != nil {
return err
}
if err = arch.WriteHeader(&tar.Header{
Name: layer + ".fs",
Size: stat.Size(),
}); err != nil {
return err
}
_, err = io.Copy(arch, f)
if err != nil {
return err
}
}
return nil
}

View File

@ -12,20 +12,22 @@ var (
) )
func casCleaner() { func casCleaner() {
for { for range time.Tick(*cacheCleanDelay) {
if !wPublicState.Get().Store.Open {
continue
}
err := cleanCAS() err := cleanCAS()
if err != nil { if err != nil {
log.Print("warn: couldn't clean cache: ", err) log.Print("warn: couldn't clean cache: ", err)
} }
time.Sleep(*cacheCleanDelay)
} }
} }
func cleanCAS() error { func cleanCAS() error {
cfg, err := readConfig() cfg, err := readConfig()
if err != nil { if err != nil {
return err return nil
} }
activeTags := make([]string, len(cfg.Hosts)) activeTags := make([]string, len(cfg.Hosts))

View File

@ -0,0 +1,185 @@
package main
import (
"crypto/rand"
"encoding/base32"
"encoding/json"
"fmt"
"log"
"path"
"strconv"
"strings"
cfsslconfig "github.com/cloudflare/cfssl/config"
"github.com/cloudflare/cfssl/csr"
yaml "gopkg.in/yaml.v2"
"novit.tech/direktil/pkg/bootstrapconfig"
"novit.tech/direktil/pkg/config"
)
func templateFuncs(sslCfg *cfsslconfig.Config) map[string]any {
getKeyCert := func(cluster, caName, name, profile, label, reqJson string) (kc KeyCert, err error) {
certReq := &csr.CertificateRequest{
KeyRequest: csr.NewKeyRequest(),
}
err = json.Unmarshal([]byte(reqJson), certReq)
if err != nil {
log.Print("CSR unmarshal failed on: ", reqJson)
return
}
return getUsableKeyCert(cluster, caName, name, profile, label, certReq, sslCfg)
}
hash := func(plain, seed []byte, hashAlg string) (hashed string, err error) {
switch hashAlg {
case "sha512crypt":
return sha512crypt(plain, seed)
case "bootstrap":
return bootstrapconfig.JoinSeedAndHash(seed, bootstrapconfig.PasswordHashFromSeed(seed, plain)), nil
default:
return "", fmt.Errorf("unknown hash alg: %q", hashAlg)
}
}
return map[string]any{
"quote": strconv.Quote,
"password": func(cluster, name, hashAlg string) (password string, err error) {
key := cluster + "/" + name
seed, err := seeds.GetOrCreate(key, func() (seed []byte, err error) {
seed = make([]byte, 16)
_, err = rand.Read(seed)
return
})
if err != nil {
return "", fmt.Errorf("failed to get seed: %w", err)
}
password, err = clusterPasswords.GetOrCreate(key, func() (password string, err error) {
raw := make([]byte, 10)
_, err = rand.Read(raw)
if err != nil {
return "", fmt.Errorf("failed to generate password: %w", err)
}
password = strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(raw))
return
})
if err != nil {
return
}
return hash([]byte(password), seed, hashAlg)
},
"token": getOrCreateClusterToken,
"ca_key": func(cluster, name string) (s string, err error) {
ca, err := getUsableClusterCA(cluster, name)
if err != nil {
return
}
s = string(ca.Key)
return
},
"ca_crt": func(cluster, name string) (s string, err error) {
ca, err := getUsableClusterCA(cluster, name)
if err != nil {
return
}
s = string(ca.Cert)
return
},
"ca_dir": func(cluster, name string) (s string, err error) {
ca, err := getUsableClusterCA(cluster, name)
if err != nil {
return
}
dir := "/etc/tls-ca/" + name
return asYaml([]config.FileDef{
{
Path: path.Join(dir, "ca.crt"),
Mode: 0644,
Content: string(ca.Cert),
},
{
Path: path.Join(dir, "ca.key"),
Mode: 0600,
Content: string(ca.Key),
},
})
},
"tls_key": func(cluster, caName, name, profile, label, reqJson string) (s string, err error) {
kc, err := getKeyCert(cluster, caName, name, profile, label, reqJson)
if err != nil {
return
}
s = string(kc.Key)
return
},
"tls_crt": func(cluster, caName, name, profile, label, reqJson string) (s string, err error) {
kc, err := getKeyCert(cluster, caName, name, profile, label, reqJson)
if err != nil {
return
}
s = string(kc.Cert)
return
},
"tls_dir": func(dir, cluster, caName, name, profile, label, reqJson string) (s string, err error) {
ca, err := getUsableClusterCA(cluster, caName)
if err != nil {
return
}
kc, err := getKeyCert(cluster, caName, name, profile, label, reqJson)
if err != nil {
return
}
return asYaml([]config.FileDef{
{
Path: path.Join(dir, "ca.crt"),
Mode: 0644,
Content: string(ca.Cert),
},
{
Path: path.Join(dir, "tls.crt"),
Mode: 0644,
Content: string(kc.Cert),
},
{
Path: path.Join(dir, "tls.key"),
Mode: 0600,
Content: string(kc.Key),
},
})
},
}
}
func asYaml(v interface{}) (string, error) {
ba, err := yaml.Marshal(v)
if err != nil {
return "", err
}
return string(ba), nil
}

View File

@ -4,7 +4,7 @@ import (
"flag" "flag"
"path/filepath" "path/filepath"
"novit.nc/direktil/pkg/localconfig" "novit.tech/direktil/pkg/localconfig"
) )
var ( var (

View File

@ -0,0 +1,3 @@
package main
var hostDownloadTokens = KVSecrets[string]{"hosts/download-tokens"}

View File

@ -0,0 +1,16 @@
package main
import (
"net/http"
"m.cluseau.fr/go/httperr"
)
var (
ErrNotFound = httperr.NotFound
ErrUnauthorized = httperr.StdStatus(http.StatusUnauthorized)
ErrForbidden = httperr.StdStatus(http.StatusForbidden)
ErrInternal = httperr.StdStatus(http.StatusInternalServerError)
ErrInvalidToken = httperr.NewStd(1000, http.StatusForbidden, "invalid token")
ErrStoreLocked = httperr.NewStd(1001, http.StatusServiceUnavailable, "store is locked")
)

View File

@ -2,13 +2,9 @@ package main
import ( import (
"encoding/json" "encoding/json"
"fmt"
"io"
"log" "log"
"net/http" "net/http"
"os"
cpio "github.com/cavaliercoder/go-cpio"
yaml "gopkg.in/yaml.v2" yaml "gopkg.in/yaml.v2"
) )
@ -28,84 +24,3 @@ func renderConfig(w http.ResponseWriter, r *http.Request, ctx *renderContext, as
return nil return nil
} }
func buildInitrd(out io.Writer, ctx *renderContext) error {
_, cfg, err := ctx.Config()
if err != nil {
return err
}
// send initrd basis
initrdPath, err := ctx.distFetch("initrd", ctx.Host.Initrd)
if err != nil {
return err
}
err = writeFile(out, initrdPath)
if err != nil {
return err
}
// and our extra archive
archive := cpio.NewWriter(out)
// - required dirs
for _, dir := range []string{
"boot",
"boot/current",
"boot/current/layers",
} {
archive.WriteHeader(&cpio.Header{
Name: dir,
Mode: 0600 | cpio.ModeDir,
})
}
// - the layers
for _, layer := range cfg.Layers {
layerVersion := ctx.Host.Versions[layer]
if layerVersion == "" {
return fmt.Errorf("layer %q not mapped to a version", layer)
}
path, err := ctx.distFetch("layers", layer, layerVersion)
if err != nil {
return err
}
stat, err := os.Stat(path)
if err != nil {
return err
}
archive.WriteHeader(&cpio.Header{
Name: "boot/current/layers/" + layer + ".fs",
Mode: 0600,
Size: stat.Size(),
})
if err = writeFile(archive, path); err != nil {
return err
}
}
// - the configuration
ba, err := yaml.Marshal(cfg)
if err != nil {
return err
}
archive.WriteHeader(&cpio.Header{
Name: "boot/config.yaml",
Mode: 0600,
Size: int64(len(ba)),
})
archive.Write(ba)
// finalize the archive
archive.Flush()
archive.Close()
return nil
}

View File

@ -1,8 +1,10 @@
package main package main
import ( import (
"io"
"log" "log"
"net/http" "net/http"
"os"
) )
func renderKernel(w http.ResponseWriter, r *http.Request, ctx *renderContext) error { func renderKernel(w http.ResponseWriter, r *http.Request, ctx *renderContext) error {
@ -15,3 +17,19 @@ func renderKernel(w http.ResponseWriter, r *http.Request, ctx *renderContext) er
http.ServeFile(w, r, path) http.ServeFile(w, r, path)
return nil return nil
} }
func fetchKernel(out io.Writer, ctx *renderContext) (err error) {
path, err := ctx.distFetch("kernels", ctx.Host.Kernel)
if err != nil {
return err
}
in, err := os.Open(path)
if err != nil {
return
}
defer in.Close()
_, err = io.Copy(out, in)
return
}

View File

@ -4,44 +4,86 @@ import (
"flag" "flag"
"log" "log"
"net/http" "net/http"
"os"
"path/filepath" "path/filepath"
restful "github.com/emicklei/go-restful" restful "github.com/emicklei/go-restful"
"github.com/mcluseau/go-swagger-ui" swaggerui "github.com/mcluseau/go-swagger-ui"
"novit.nc/direktil/pkg/cas" "m.cluseau.fr/go/watchable/streamsse"
"novit.nc/direktil/local-server/pkg/apiutils" "novit.tech/direktil/pkg/cas"
dlshtml "novit.tech/direktil/local-server/html"
"novit.tech/direktil/local-server/pkg/apiutils"
) )
const ( const (
etcDir = "/etc/direktil" etcDir = "/etc/direktil"
) )
var Version = "dev"
var ( var (
address = flag.String("address", ":7606", "HTTP listen address") address = flag.String("address", ":7606", "HTTP listen address")
tlsAddress = flag.String("tls-address", "", "HTTPS listen address") tlsAddress = flag.String("tls-address", "", "HTTPS listen address")
certFile = flag.String("tls-cert", etcDir+"/server.crt", "Server TLS certificate") certFile = flag.String("tls-cert", etcDir+"/server.crt", "Server TLS certificate")
keyFile = flag.String("tls-key", etcDir+"/server.key", "Server TLS key") keyFile = flag.String("tls-key", etcDir+"/server.key", "Server TLS key")
autoUnlock = flag.String("auto-unlock", "", "Auto-unlock store (testing only!) env: DLS_AUTO_UNLOCK")
casStore cas.Store casStore cas.Store
) )
func main() { func main() {
log.SetFlags(log.LstdFlags | log.Lshortfile)
flag.Parse() flag.Parse()
if *address == "" && *tlsAddress == "" { if *address == "" && *tlsAddress == "" {
log.Fatal("no listen address given") log.Fatal("no listen address given")
} }
log.Print("Direktil local-server version ", Version)
wPublicState.Change(func(s *PublicState) { s.ServerVersion = Version })
computeUIHash()
openSecretStore()
{
autoUnlock := *autoUnlock
if autoUnlock == "" {
autoUnlock = os.Getenv("DLS_AUTO_UNLOCK")
}
if autoUnlock != "" {
log.Printf("auto-unlocking the store")
err := unlockSecretStore("test", []byte(autoUnlock))
if err.Any() {
log.Fatal(err)
}
log.Print("store auto-unlocked, token is ", adminToken)
}
os.Setenv("DLS_AUTO_UNLOCK", "")
}
casStore = cas.NewDir(filepath.Join(*dataDir, "cache")) casStore = cas.NewDir(filepath.Join(*dataDir, "cache"))
go casCleaner() go casCleaner()
apiutils.Setup(func() { apiutils.Setup(func() {
restful.Add(buildWS()) registerWS(restful.DefaultContainer)
}) })
swaggerui.HandleAt("/swagger-ui/") swaggerui.HandleAt("/swagger-ui/")
staticHandler := http.FileServer(http.FS(dlshtml.FS))
http.Handle("/favicon.ico", staticHandler)
http.Handle("/ui/", staticHandler)
http.Handle("/public-state", streamsse.StreamHandler(wPublicState))
http.Handle("/state", requireAdmin(streamsse.StreamHandler(wState)))
if *address != "" { if *address != "" {
log.Print("HTTP listening on ", *address) log.Print("HTTP listening on ", *address)
go log.Fatal(http.ListenAndServe(*address, nil)) go log.Fatal(http.ListenAndServe(*address, nil))

View File

@ -4,36 +4,49 @@ import (
"bytes" "bytes"
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
"encoding/json" "fmt"
"io" "io"
"log" "log"
"net/http" "net/http"
"net/url"
"path" "path"
"path/filepath" "path/filepath"
"text/template" "text/template"
cfsslconfig "github.com/cloudflare/cfssl/config" cfsslconfig "github.com/cloudflare/cfssl/config"
"github.com/cloudflare/cfssl/csr" restful "github.com/emicklei/go-restful"
yaml "gopkg.in/yaml.v2" yaml "gopkg.in/yaml.v2"
"novit.nc/direktil/pkg/config" "novit.tech/direktil/pkg/config"
"novit.nc/direktil/pkg/localconfig" "novit.tech/direktil/pkg/localconfig"
bsconfig "novit.tech/direktil/pkg/bootstrapconfig"
) )
var cmdlineParam = restful.QueryParameter("cmdline", "Linux kernel cmdline addition")
type renderContext struct { type renderContext struct {
Host *localconfig.Host Host *localconfig.Host
SSLConfig string SSLConfig *cfsslconfig.Config
// Linux kernel extra cmdline
CmdLine string `yaml:"-"`
} }
func renderCtx(w http.ResponseWriter, r *http.Request, ctx *renderContext, what string, func renderCtx(w http.ResponseWriter, r *http.Request, ctx *renderContext, what string,
create func(out io.Writer, ctx *renderContext) error) error { create func(out io.Writer, ctx *renderContext) error) error {
log.Printf("sending %s for %q", what, ctx.Host.Name)
tag, err := ctx.Tag() tag, err := ctx.Tag()
if err != nil { if err != nil {
return err return err
} }
ctx.CmdLine = r.URL.Query().Get(cmdlineParam.Data().Name)
if ctx.CmdLine != "" {
what = what + "?cmdline=" + url.QueryEscape(ctx.CmdLine)
}
// get it or create it // get it or create it
content, meta, err := casStore.GetOrCreate(tag, what, func(out io.Writer) error { content, meta, err := casStore.GetOrCreate(tag, what, func(out io.Writer) error {
log.Printf("building %s for %q", what, ctx.Host.Name) log.Printf("building %s for %q", what, ctx.Host.Name)
@ -45,43 +58,67 @@ func renderCtx(w http.ResponseWriter, r *http.Request, ctx *renderContext, what
} }
// serve it // serve it
log.Printf("sending %s for %q", what, ctx.Host.Name)
http.ServeContent(w, r, what, meta.ModTime(), content) http.ServeContent(w, r, what, meta.ModTime(), content)
return nil return nil
} }
var prevSSLConfig = "-" func sslConfigFromLocalConfig(cfg *localconfig.Config) (sslCfg *cfsslconfig.Config, err error) {
if len(cfg.SSLConfig) == 0 {
func newRenderContext(host *localconfig.Host, cfg *localconfig.Config) (ctx *renderContext, err error) { sslCfg = &cfsslconfig.Config{}
if prevSSLConfig != cfg.SSLConfig { } else {
var sslCfg *cfsslconfig.Config sslCfg, err = cfsslconfig.LoadConfig([]byte(cfg.SSLConfig))
if len(cfg.SSLConfig) == 0 {
sslCfg = &cfsslconfig.Config{}
} else {
sslCfg, err = cfsslconfig.LoadConfig([]byte(cfg.SSLConfig))
if err != nil {
return
}
}
err = loadSecretData(sslCfg)
if err != nil { if err != nil {
return return
} }
}
return
}
prevSSLConfig = cfg.SSLConfig func newRenderContext(host *localconfig.Host, cfg *localconfig.Config) (ctx *renderContext, err error) {
sslCfg, err := sslConfigFromLocalConfig(cfg)
if err != nil {
return
} }
return &renderContext{ return &renderContext{
SSLConfig: cfg.SSLConfig,
Host: host, Host: host,
SSLConfig: sslCfg,
}, nil }, nil
} }
func (ctx *renderContext) Config() (ba []byte, cfg *config.Config, err error) { func (ctx *renderContext) Config() (ba []byte, cfg *config.Config, err error) {
ba, err = ctx.render(ctx.Host.Config)
if err != nil {
return
}
cfg = &config.Config{}
if err = yaml.Unmarshal(ba, cfg); err != nil {
return
}
return
}
func (ctx *renderContext) BootstrapConfig() (ba []byte, cfg *bsconfig.Config, err error) {
ba, err = ctx.render(ctx.Host.BootstrapConfig)
if err != nil {
return
}
cfg = &bsconfig.Config{}
if err = yaml.Unmarshal(ba, cfg); err != nil {
return
}
return
}
func (ctx *renderContext) render(templateText string) (ba []byte, err error) {
tmpl, err := template.New(ctx.Host.Name + "/config"). tmpl, err := template.New(ctx.Host.Name + "/config").
Funcs(ctx.templateFuncs()). Funcs(ctx.TemplateFuncs()).
Parse(ctx.Host.Config) Parse(templateText)
if err != nil { if err != nil {
return return
@ -92,147 +129,10 @@ func (ctx *renderContext) Config() (ba []byte, cfg *config.Config, err error) {
return return
} }
if secretData.Changed() {
err = secretData.Save()
if err != nil {
return
}
}
ba = buf.Bytes() ba = buf.Bytes()
cfg = &config.Config{}
if err = yaml.Unmarshal(buf.Bytes(), cfg); err != nil {
return
}
return return
} }
func (ctx *renderContext) templateFuncs() map[string]interface{} {
getKeyCert := func(cluster, caName, name, profile, label, reqJson string) (kc *KeyCert, err error) {
certReq := &csr.CertificateRequest{
KeyRequest: csr.NewBasicKeyRequest(),
}
err = json.Unmarshal([]byte(reqJson), certReq)
if err != nil {
log.Print("CSR unmarshal failed on: ", reqJson)
return
}
return secretData.KeyCert(cluster, caName, name, profile, label, certReq)
}
asYaml := func(v interface{}) (string, error) {
ba, err := yaml.Marshal(v)
if err != nil {
return "", err
}
return string(ba), nil
}
return map[string]interface{}{
"token": func(cluster, name string) (s string, err error) {
return secretData.Token(cluster, name)
},
"ca_key": func(cluster, name string) (s string, err error) {
ca, err := secretData.CA(cluster, name)
if err != nil {
return
}
s = string(ca.Key)
return
},
"ca_crt": func(cluster, name string) (s string, err error) {
ca, err := secretData.CA(cluster, name)
if err != nil {
return
}
s = string(ca.Cert)
return
},
"ca_dir": func(cluster, name string) (s string, err error) {
ca, err := secretData.CA(cluster, name)
if err != nil {
return
}
dir := "/etc/tls-ca/" + name
return asYaml([]config.FileDef{
{
Path: path.Join(dir, "ca.crt"),
Mode: 0644,
Content: string(ca.Cert),
},
{
Path: path.Join(dir, "ca.key"),
Mode: 0600,
Content: string(ca.Key),
},
})
},
"tls_key": func(cluster, caName, name, profile, label, reqJson string) (s string, err error) {
kc, err := getKeyCert(cluster, caName, name, profile, label, reqJson)
if err != nil {
return
}
s = string(kc.Key)
return
},
"tls_crt": func(cluster, caName, name, profile, label, reqJson string) (s string, err error) {
kc, err := getKeyCert(cluster, caName, name, profile, label, reqJson)
if err != nil {
return
}
s = string(kc.Cert)
return
},
"tls_dir": func(dir, cluster, caName, name, profile, label, reqJson string) (s string, err error) {
ca, err := secretData.CA(cluster, caName)
if err != nil {
return
}
kc, err := getKeyCert(cluster, caName, name, profile, label, reqJson)
if err != nil {
return
}
return asYaml([]config.FileDef{
{
Path: path.Join(dir, "ca.crt"),
Mode: 0644,
Content: string(ca.Cert),
},
{
Path: path.Join(dir, "tls.crt"),
Mode: 0644,
Content: string(kc.Cert),
},
{
Path: path.Join(dir, "tls.key"),
Mode: 0600,
Content: string(kc.Key),
},
})
},
}
}
func (ctx *renderContext) distFilePath(path ...string) string { func (ctx *renderContext) distFilePath(path ...string) string {
return filepath.Join(append([]string{*dataDir, "dist"}, path...)...) return filepath.Join(append([]string{*dataDir, "dist"}, path...)...)
} }
@ -270,3 +170,69 @@ func asMap(v interface{}) map[string]interface{} {
return result return result
} }
func (ctx *renderContext) TemplateFuncs() map[string]any {
funcs := templateFuncs(ctx.SSLConfig)
for name, method := range map[string]any{
"ssh_host_keys": func(dir, cluster, host string) (s string, err error) {
if host == "" {
host = ctx.Host.Name
}
if host != ctx.Host.Name {
err = fmt.Errorf("wrong host name")
return
}
pairs, err := getSSHKeyPairs(host)
if err != nil {
return
}
files := make([]config.FileDef, 0, len(pairs)*2)
for _, pair := range pairs {
basePath := path.Join(dir, "ssh_host_"+pair.Type+"_key")
files = append(files, []config.FileDef{
{
Path: basePath,
Mode: 0600,
Content: pair.Private,
},
{
Path: basePath + ".pub",
Mode: 0644,
Content: pair.Public,
},
}...)
}
return asYaml(files)
},
"host_download_token": func() (token string, err error) {
key := ctx.Host.Name
token, found, err := hostDownloadTokens.Get(key)
if err != nil {
return
}
if !found {
token, err = newToken(32)
if err != nil {
return
}
err = hostDownloadTokens.Put(key, token)
if err != nil {
return
}
}
return
},
} {
funcs[name] = method
}
return funcs
}

View File

@ -0,0 +1,350 @@
package main
import (
"encoding/json"
"log"
"net/http"
"os"
"path/filepath"
"sort"
"strings"
"sync"
restful "github.com/emicklei/go-restful"
"m.cluseau.fr/go/httperr"
"novit.tech/direktil/local-server/secretstore"
)
var secStore *secretstore.Store
func secStoreRoot() string { return filepath.Join(*dataDir, "secrets") }
func secStorePath(name string) string { return filepath.Join(secStoreRoot(), name) }
func secKeysStorePath() string { return secStorePath(".keys") }
func openSecretStore() {
var err error
keysPath := secKeysStorePath()
if err := os.MkdirAll(filepath.Dir(filepath.Dir(keysPath)), 0755); err != nil {
log.Fatal("failed to create dirs: ", err)
}
if err := os.MkdirAll(filepath.Dir(keysPath), 0700); err != nil {
log.Fatal("failed to secret store dir: ", err)
}
secStore, err = secretstore.Open(keysPath)
switch {
case err == nil:
wPublicState.Change(func(v *PublicState) {
v.Store.New = false
v.Store.Open = false
})
case os.IsNotExist(err):
secStore = secretstore.New()
wPublicState.Change(func(v *PublicState) {
v.Store.New = true
v.Store.Open = false
})
default:
log.Fatal("failed to open keys store: ", err)
}
}
var (
unlockMutex = sync.Mutex{}
ErrStoreAlreadyUnlocked = httperr.NewStd(1, http.StatusConflict, "store already unlocked")
ErrInvalidPassphrase = httperr.NewStd(2, http.StatusBadRequest, "invalid passphrase")
)
func unlockSecretStore(name string, passphrase []byte) (err httperr.Error) {
unlockMutex.Lock()
defer unlockMutex.Unlock()
if secStore.Unlocked() {
return ErrStoreAlreadyUnlocked
}
if secStore.IsNew() {
err := secStore.Init(name, passphrase)
if err != nil {
return httperr.Internal(err)
}
err = secStore.SaveTo(secKeysStorePath())
if err != nil {
log.Print("secret store save error: ", err)
secStore.Close()
return httperr.Internal(err)
}
} else {
if !secStore.Unlock([]byte(passphrase)) {
return ErrInvalidPassphrase
}
}
token := ""
if err := readSecret("admin-token", &token); err != nil {
if !os.IsNotExist(err) {
log.Print("failed to read admin token: ", err)
secStore.Close()
return httperr.Internal(err)
}
token, err = newToken(32)
if err != nil {
secStore.Close()
return httperr.Internal(err)
}
err = writeSecret("admin-token", token)
if err != nil {
log.Print("write error: ", err)
secStore.Close()
return httperr.Internal(err)
}
log.Print("wrote new admin token")
}
adminToken = token
{
token, err := newToken(16)
if err != nil {
secStore.Close()
return httperr.Internal(err)
}
wState.Change(func(v *State) {
v.Store.DownloadToken = token
})
}
wPublicState.Change(func(v *PublicState) {
v.Store.New = false
v.Store.Open = true
})
go updateState()
go migrateSecrets()
return
}
func readSecret(name string, value any) (err error) {
f, err := os.Open(secStorePath(name + ".data"))
if err != nil {
return
}
defer f.Close()
in, err := secStore.NewReader(f)
if err != nil {
return
}
return json.NewDecoder(in).Decode(value)
}
func writeSecret(name string, value any) (err error) {
path := secStorePath(name + ".data.new")
if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil {
return
}
f, err := os.Create(path)
if err != nil {
return
}
err = func() (err error) {
defer f.Close()
out, err := secStore.NewWriter(f)
if err != nil {
return
}
return json.NewEncoder(out).Encode(value)
}()
if err != nil {
return
}
err = os.Rename(f.Name(), secStorePath(name+".data"))
if err != nil {
return
}
go updateState()
return
}
var secL sync.Mutex
func updateSecret[T any](name string, update func(*T)) (err error) {
secL.Lock()
defer secL.Unlock()
v := new(T)
err = readSecret(name, v)
if err != nil {
if !os.IsNotExist(err) {
return
}
err = nil
}
update(v)
return writeSecret(name, *v)
}
func updateSecretWithKey[T any](name, key string, update func(v *T)) (err error) {
secL.Lock()
defer secL.Unlock()
kvs := map[string]*T{}
err = readSecret(name, &kvs)
if err != nil {
if !os.IsNotExist(err) {
return
}
err = nil
}
update(kvs[key])
return writeSecret(name, kvs)
}
type KVSecrets[T any] struct{ Name string }
func (s KVSecrets[T]) Data() (kvs map[string]T, err error) {
kvs = make(map[string]T)
err = readSecret(s.Name, &kvs)
if err != nil {
if !os.IsNotExist(err) {
return
}
err = nil
}
return
}
func (s KVSecrets[T]) Keys(prefix string) (keys []string, err error) {
kvs, err := s.Data()
if err != nil {
return
}
keys = make([]string, 0, len(kvs))
for k := range kvs {
if !strings.HasPrefix(k, prefix) {
continue
}
keys = append(keys, k[len(prefix):])
}
sort.Strings(keys)
return
}
func (s KVSecrets[T]) Get(key string) (v T, found bool, err error) {
kvs, err := s.Data()
if err != nil {
return
}
v, found = kvs[key]
return
}
func (s KVSecrets[T]) Put(key string, v T) (err error) {
secL.Lock()
defer secL.Unlock()
kvs, err := s.Data()
if err != nil {
return
}
kvs[key] = v
err = writeSecret(s.Name, kvs)
return
}
func (s KVSecrets[T]) GetOrCreate(key string, create func() (T, error)) (v T, err error) {
v, found, err := s.Get(key)
if err != nil {
return
}
if !found {
v, err = create()
if err != nil {
return
}
err = s.Put(key, v)
}
return
}
func (s KVSecrets[T]) WsList(resp *restful.Response, prefix string) {
keys, err := s.Keys(prefix)
if err != nil {
wsError(resp, err)
return
}
resp.WriteEntity(keys)
}
func (s KVSecrets[T]) WsGet(resp *restful.Response, key string) {
keys, found, err := s.Get(key)
if err != nil {
wsError(resp, err)
return
}
if !found {
wsNotFound(resp)
return
}
resp.WriteEntity(keys)
}
func (s KVSecrets[T]) WsPut(req *restful.Request, resp *restful.Response, key string) {
v := new(T)
err := req.ReadEntity(v)
if err != nil {
wsBadRequest(resp, err.Error())
return
}
err = s.Put(key, *v)
if err != nil {
wsError(resp, err)
return
}
}

View File

@ -0,0 +1,88 @@
package main
import (
"log"
"os"
cfsslconfig "github.com/cloudflare/cfssl/config"
)
func migrateSecrets() {
if _, err := os.Stat(secretDataPath()); err != nil {
if os.IsNotExist(err) {
return
}
log.Print("not migrating old secrets: ", err)
return
}
log.Print("migrating old secrets")
log := log.New(log.Default().Writer(), "secrets migration: ", log.Flags()|log.Lmsgprefix)
// load secrets
cfg, err := readConfig()
if err != nil {
log.Fatal(err)
return
}
var sslCfg *cfsslconfig.Config
if len(cfg.SSLConfig) == 0 {
sslCfg = &cfsslconfig.Config{}
} else {
sslCfg, err = cfsslconfig.LoadConfig([]byte(cfg.SSLConfig))
if err != nil {
return
}
}
secretData, err := loadSecretData(sslCfg)
if err != nil {
log.Fatal(err)
return
}
for clusterName, cluster := range secretData.clusters {
for k, v := range cluster.Tokens {
err = clusterTokens.Put(clusterName+"/"+k, v)
if err != nil {
log.Fatal(err)
return
}
}
for k, v := range cluster.Passwords {
err = clusterPasswords.Put(clusterName+"/"+k, v)
if err != nil {
log.Fatal(err)
return
}
}
for caName, ca := range cluster.CAs {
clusterCAs.Put(clusterName+"/"+caName, CA{Key: ca.Key, Cert: ca.Cert})
for signedName, signed := range ca.Signed {
err = clusterCASignedKeys.Put(clusterName+"/"+caName+"/"+signedName, *signed)
if err != nil {
log.Fatal(err)
}
}
}
for hostName, pairs := range cluster.SSHKeyPairs {
err = sshHostKeys.Put(hostName, pairs)
if err != nil {
log.Fatal(err)
}
}
}
if err := os.Rename(secretDataPath(), secretDataPath()+".migrated"); err != nil {
log.Fatal("failed to rename migrated secrets: ", err)
}
}

View File

@ -1,50 +1,28 @@
package main package main
import ( import (
"crypto/rand"
"encoding/base32"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt"
"io/ioutil" "io/ioutil"
"net"
"os" "os"
"path/filepath" "path/filepath"
"sync" "time"
"github.com/cloudflare/cfssl/certinfo"
"github.com/cloudflare/cfssl/config" "github.com/cloudflare/cfssl/config"
"github.com/cloudflare/cfssl/csr"
"github.com/cloudflare/cfssl/helpers"
"github.com/cloudflare/cfssl/initca"
"github.com/cloudflare/cfssl/log" "github.com/cloudflare/cfssl/log"
"github.com/cloudflare/cfssl/signer"
"github.com/cloudflare/cfssl/signer/local"
"k8s.io/apimachinery/pkg/util/validation"
"k8s.io/apimachinery/pkg/util/validation/field"
)
var (
secretData *SecretData
) )
type SecretData struct { type SecretData struct {
l sync.Mutex
clusters map[string]*ClusterSecrets clusters map[string]*ClusterSecrets
changed bool
config *config.Config config *config.Config
} }
type ClusterSecrets struct { type ClusterSecrets struct {
CAs map[string]*CA CAs map[string]*CA
Tokens map[string]string Tokens map[string]string
} Passwords map[string]string
SSHKeyPairs map[string][]SSHKeyPair
type CA struct {
Key []byte
Cert []byte
Signed map[string]*KeyCert
} }
type KeyCert struct { type KeyCert struct {
@ -57,21 +35,18 @@ func secretDataPath() string {
return filepath.Join(*dataDir, "secret-data.json") return filepath.Join(*dataDir, "secret-data.json")
} }
func loadSecretData(config *config.Config) (err error) { func loadSecretData(config *config.Config) (sd *SecretData, err error) {
log.Info("Loading secret data") log.Info("Loading secret data")
sd := &SecretData{ sd = &SecretData{
clusters: make(map[string]*ClusterSecrets), clusters: make(map[string]*ClusterSecrets),
changed: false,
config: config, config: config,
} }
ba, err := ioutil.ReadFile(secretDataPath()) ba, err := ioutil.ReadFile(secretDataPath())
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
sd.changed = true
err = nil err = nil
secretData = sd
return return
} }
return return
@ -81,207 +56,21 @@ func loadSecretData(config *config.Config) (err error) {
return return
} }
secretData = sd
return return
} }
func (sd *SecretData) Changed() bool { func checkCertUsable(certPEM []byte) error {
return sd.changed cert, err := certinfo.ParseCertificatePEM(certPEM)
}
func (sd *SecretData) Save() error {
sd.l.Lock()
defer sd.l.Unlock()
log.Info("Saving secret data")
ba, err := json.Marshal(sd.clusters)
if err != nil { if err != nil {
return err return err
} }
return ioutil.WriteFile(secretDataPath(), ba, 0600)
} certDuration := cert.NotAfter.Sub(cert.NotBefore)
delayBeforeRegen := certDuration / 3 // TODO allow configuration
func (sd *SecretData) cluster(name string) (cs *ClusterSecrets) {
cs, ok := sd.clusters[name] if cert.NotAfter.Sub(time.Now()) < delayBeforeRegen {
if ok { return errors.New("too old")
return }
}
return nil
sd.l.Lock()
defer sd.l.Unlock()
log.Info("secret-data: new cluster: ", name)
cs = &ClusterSecrets{
CAs: make(map[string]*CA),
Tokens: make(map[string]string),
}
sd.clusters[name] = cs
sd.changed = true
return
}
func (sd *SecretData) Token(cluster, name string) (token string, err error) {
cs := sd.cluster(cluster)
token = cs.Tokens[name]
if token != "" {
return
}
sd.l.Lock()
defer sd.l.Unlock()
log.Info("secret-data: new token in cluster ", cluster, ": ", name)
b := make([]byte, 16)
_, err = rand.Read(b)
if err != nil {
return
}
token = base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(b)
cs.Tokens[name] = token
sd.changed = true
return
}
func (sd *SecretData) CA(cluster, name string) (ca *CA, err error) {
cs := sd.cluster(cluster)
ca, ok := cs.CAs[name]
if ok {
return
}
sd.l.Lock()
defer sd.l.Unlock()
log.Info("secret-data: new CA in cluster ", cluster, ": ", name)
req := &csr.CertificateRequest{
CN: "Direktil Local Server",
KeyRequest: &csr.BasicKeyRequest{
A: "ecdsa",
S: 521, // 256, 384, 521
},
Names: []csr.Name{
{
C: "NC",
O: "novit.nc",
},
},
}
cert, _, key, err := initca.New(req)
if err != nil {
return
}
ca = &CA{
Key: key,
Cert: cert,
Signed: make(map[string]*KeyCert),
}
cs.CAs[name] = ca
sd.changed = true
return
}
func (sd *SecretData) KeyCert(cluster, caName, name, profile, label string, req *csr.CertificateRequest) (kc *KeyCert, err error) {
for idx, host := range req.Hosts {
if ip := net.ParseIP(host); ip != nil {
// valid IP (v4 or v6)
continue
}
if host == "*" {
continue
}
if errs := validation.IsDNS1123Subdomain(host); len(errs) == 0 {
continue
}
if errs := validation.IsWildcardDNS1123Subdomain(host); len(errs) == 0 {
continue
}
path := field.NewPath(cluster, name, "hosts").Index(idx)
return nil, fmt.Errorf("%v: %q is not an IP or FQDN", path, host)
}
if req.CA != nil {
err = errors.New("no CA section allowed here")
return
}
ca, err := sd.CA(cluster, caName)
if err != nil {
return
}
rh := hash(req)
kc, ok := ca.Signed[name]
if ok && rh == kc.ReqHash {
return
} else if ok {
log.Infof("secret-data: cluster %s: CA %s: CSR changed for %s: hash=%q previous=%q",
cluster, caName, name, rh, kc.ReqHash)
} else {
log.Infof("secret-data: cluster %s: CA %s: new CSR for %s", cluster, caName, name)
}
sd.l.Lock()
defer sd.l.Unlock()
sgr, err := ca.Signer(sd.config.Signing)
if err != nil {
return
}
generator := &csr.Generator{Validator: func(_ *csr.CertificateRequest) error { return nil }}
csr, key, err := generator.ProcessRequest(req)
if err != nil {
return
}
signReq := signer.SignRequest{
Request: string(csr),
Profile: profile,
Label: label,
}
cert, err := sgr.Sign(signReq)
if err != nil {
return
}
kc = &KeyCert{
Key: key,
Cert: cert,
ReqHash: rh,
}
ca.Signed[name] = kc
sd.changed = true
return
}
func (ca *CA) Signer(policy *config.Signing) (result *local.Signer, err error) {
caCert, err := helpers.ParseCertificatePEM(ca.Cert)
if err != nil {
return
}
caKey, err := helpers.ParsePrivateKeyPEM(ca.Key)
if err != nil {
return
}
return local.NewSigner(caKey, caCert, signer.DefaultSigAlgo(caKey), policy)
} }

View File

@ -0,0 +1,39 @@
package main
import (
"encoding/base64"
crypthash "github.com/sergeymakinen/go-crypt/hash"
"github.com/sergeymakinen/go-crypt/sha512"
)
// for some reason, no implementation of crypt's sha512 is clean enough :(
func sha512crypt(password, seed []byte) (string, error) {
// loose salt entropy because of character restriction in the salt
salt := []byte(base64.RawStdEncoding.EncodeToString(seed))[:sha512.MaxSaltLength]
// - base64 allows '+' where the salt accepts '.'
for i, c := range salt {
if c == '+' {
salt[i] = '.'
}
}
scheme := struct {
HashPrefix string
Rounds uint32 `hash:"param:rounds,omitempty"`
Salt []byte
Sum [86]byte
}{
HashPrefix: sha512.Prefix,
Rounds: sha512.DefaultRounds,
Salt: salt,
}
key, err := sha512.Key([]byte(password), scheme.Salt, scheme.Rounds)
if err != nil {
return "", err
}
crypthash.LittleEndianEncoding.Encode(scheme.Sum[:], key)
return crypthash.Marshal(scheme)
}

View File

@ -0,0 +1,168 @@
package main
import (
"crypto/dsa"
"crypto/ecdsa"
"crypto/ed25519"
"crypto/elliptic"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/asn1"
"fmt"
"io/ioutil"
"os"
"os/exec"
)
var sshHostKeys = KVSecrets[[]SSHKeyPair]{"hosts/ssh-host-keys"}
type SSHKeyPair struct {
Type string
Public string
Private string
}
func getSSHKeyPairs(host string) (pairs []SSHKeyPair, err error) {
pairs, _, err = sshHostKeys.Get(host)
didGenerate := false
genLoop:
for _, keyType := range []string{
"rsa",
"dsa",
"ecdsa",
"ed25519",
} {
for _, pair := range pairs {
if pair.Type == keyType {
continue genLoop
}
}
err = func() (err error) {
outFile, err := ioutil.TempFile("/tmp", "dls-key.")
if err != nil {
return
}
outPath := outFile.Name()
removeTemp := func() {
os.Remove(outPath)
os.Remove(outPath + ".pub")
}
removeTemp()
defer removeTemp()
var out, privKey, pubKey []byte
cmd := exec.Command("ssh-keygen",
"-N", "",
"-C", "root@"+host,
"-f", outPath,
"-t", keyType)
out, err = cmd.CombinedOutput()
if err != nil {
err = fmt.Errorf("ssh-keygen failed: %v: %s", err, string(out))
return
}
privKey, err = ioutil.ReadFile(outPath)
if err != nil {
return
}
pubKey, err = ioutil.ReadFile(outPath + ".pub")
if err != nil {
return
}
pairs = append(pairs, SSHKeyPair{
Type: keyType,
Public: string(pubKey),
Private: string(privKey),
})
didGenerate = true
return
}()
if err != nil {
return
}
}
if didGenerate {
err = sshHostKeys.Put(host, pairs)
if err != nil {
return
}
}
return
}
func sshKeyGenDSA() (data []byte, pubKey interface{}, err error) {
privKey := &dsa.PrivateKey{}
err = dsa.GenerateParameters(&privKey.Parameters, rand.Reader, dsa.L1024N160)
if err != nil {
return
}
err = dsa.GenerateKey(privKey, rand.Reader)
if err != nil {
return
}
data, err = asn1.Marshal(*privKey)
//data, err = x509.MarshalPKCS8PrivateKey(privKey)
if err != nil {
return
}
pubKey = privKey.PublicKey
return
}
func sshKeyGenRSA() (data []byte, pubKey interface{}, err error) {
privKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return
}
data = x509.MarshalPKCS1PrivateKey(privKey)
pubKey = privKey.Public()
return
}
func sshKeyGenECDSA() (data []byte, pubKey interface{}, err error) {
privKey, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader)
if err != nil {
return
}
data, err = x509.MarshalPKCS8PrivateKey(privKey)
if err != nil {
return
}
pubKey = privKey.Public()
return
}
func sshKeyGenED25519() (data []byte, pubKey interface{}, err error) {
pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
data, err = x509.MarshalPKCS8PrivateKey(privKey)
if err != nil {
return
}
return
}

View File

@ -0,0 +1,10 @@
package main
import "testing"
func TestSSHKeyGet(t *testing.T) {
// TODO needs fake secret store
// if _, err := getSSHKeyPairs("host"); err != nil {
// t.Error(err)
// }
}

View File

@ -0,0 +1,138 @@
package main
import (
"log"
"m.cluseau.fr/go/watchable"
"novit.tech/direktil/pkg/localconfig"
)
type PublicState struct {
ServerVersion string
UIHash string
Store struct {
New bool
Open bool
}
}
var wPublicState = watchable.New[PublicState]()
type State struct {
HasConfig bool
Store struct {
DownloadToken string
KeyNames []string
}
Clusters []ClusterState
Hosts []HostState
Config *localconfig.Config
Downloads map[string]DownloadSpec
}
type ClusterState struct {
Name string
Addons bool
Passwords []string
Tokens []string
CAs []CAState
}
type HostState struct {
Name string
Cluster string
IPs []string
}
type CAState struct {
Name string
Signed []string
}
var wState = watchable.New[State]()
func init() {
wState.Set(State{Downloads: map[string]DownloadSpec{}})
}
func updateState() {
log.Print("updating state")
// store key names
keyNames := make([]string, 0, len(secStore.Keys))
for _, key := range secStore.Keys {
keyNames = append(keyNames, key.Name)
}
// config
cfg, err := readConfig()
if err != nil {
wState.Change(func(v *State) { v.HasConfig = false; v.Config = nil; v.Store.KeyNames = keyNames })
return
}
if secStore.IsNew() || !secStore.Unlocked() {
wState.Change(func(v *State) { v.HasConfig = false; v.Config = nil; v.Store.KeyNames = keyNames })
return
}
// remove heavy data
clusters := make([]ClusterState, 0, len(cfg.Clusters))
for _, cluster := range cfg.Clusters {
c := ClusterState{
Name: cluster.Name,
Addons: len(cluster.Addons) != 0,
}
c.Passwords, err = clusterPasswords.Keys(c.Name + "/")
if err != nil {
log.Print("failed to read cluster passwords: ", err)
}
c.Tokens, err = clusterTokens.Keys(c.Name + "/")
if err != nil {
log.Print("failed to read cluster tokens: ", err)
}
caNames, err := clusterCAs.Keys(c.Name + "/")
if err != nil {
log.Print("failed to read cluster CAs: ", err)
}
for _, caName := range caNames {
ca := CAState{Name: caName}
signedNames, err := clusterCASignedKeys.Keys(c.Name + "/" + caName + "/")
if err != nil {
log.Print("failed to read cluster CA signed keys: ", err)
}
for _, signedName := range signedNames {
ca.Signed = append(ca.Signed, signedName)
}
c.CAs = append(c.CAs, ca)
}
clusters = append(clusters, c)
}
hosts := make([]HostState, 0, len(cfg.Hosts))
for _, host := range cfg.Hosts {
h := HostState{
Name: host.Name,
Cluster: host.ClusterName,
IPs: host.IPs,
}
hosts = append(hosts, h)
}
// done
wState.Change(func(v *State) {
v.HasConfig = true
v.Store.KeyNames = keyNames
v.Clusters = clusters
v.Hosts = hosts
})
}

View File

@ -0,0 +1,178 @@
package main
import (
"crypto"
"errors"
"fmt"
"log"
"net"
"github.com/cloudflare/cfssl/config"
"github.com/cloudflare/cfssl/csr"
"github.com/cloudflare/cfssl/helpers"
"github.com/cloudflare/cfssl/initca"
"github.com/cloudflare/cfssl/signer"
"github.com/cloudflare/cfssl/signer/local"
"k8s.io/apimachinery/pkg/util/validation"
)
type CA struct {
Key []byte
Cert []byte
Signed map[string]*KeyCert
}
func (ca *CA) Init() (err error) {
req := ca.newReq()
cert, _, key, err := initca.New(req)
if err != nil {
err = fmt.Errorf("initca: %w", err)
return
}
ca.Key = key
ca.Cert = cert
return
}
func (ca *CA) RenewCert() (err error) {
var signer crypto.Signer
signer, err = helpers.ParsePrivateKeyPEM(ca.Key)
if err != nil {
return
}
newCert, _, err := initca.NewFromSigner(ca.newReq(), signer)
if err != nil {
return
}
ca.Cert = newCert
return
}
func (_ CA) newReq() *csr.CertificateRequest {
return &csr.CertificateRequest{
CN: "Direktil Local Server",
KeyRequest: &csr.KeyRequest{
A: "ecdsa",
S: 521, // 256, 384, 521
},
Names: []csr.Name{
{
O: "novit.io",
},
},
}
}
func (ca CA) Signer(policy *config.Signing) (result *local.Signer, err error) {
caCert, err := helpers.ParseCertificatePEM(ca.Cert)
if err != nil {
return
}
caKey, err := helpers.ParsePrivateKeyPEM(ca.Key)
if err != nil {
return
}
return local.NewSigner(caKey, caCert, signer.DefaultSigAlgo(caKey), policy)
}
func getUsableKeyCert(cluster, caName, name, profile, label string, req *csr.CertificateRequest, cfg *config.Config) (kc KeyCert, err error) {
log := log.New(log.Default().Writer(), cluster+": CA "+caName+": ", log.Flags()|log.Lmsgprefix)
ca, err := getUsableClusterCA(cluster, caName)
if err != nil {
return
}
for _, host := range req.Hosts {
if ip := net.ParseIP(host); ip != nil {
// valid IP (v4 or v6)
continue
}
if host == "*" {
continue
}
if errs := validation.IsDNS1123Subdomain(host); len(errs) == 0 {
continue
}
if errs := validation.IsWildcardDNS1123Subdomain(host); len(errs) == 0 {
continue
}
err = fmt.Errorf("%q is not an IP or FQDN", host)
return
}
if req.CA != nil {
err = errors.New("no CA section allowed here")
return
}
rh := hash(req)
key := cluster + "/" + caName + "/" + name
kc, found, err := clusterCASignedKeys.Get(key)
if err != nil {
return
}
if found {
if rh == kc.ReqHash {
err = checkCertUsable(kc.Cert)
if err == nil {
return // all good, no need to create or renew
}
log.Print("regenerating certificate: ", err)
} else {
log.Printf("CSR changed for %s: hash=%q previous=%q", name, rh, kc.ReqHash)
}
} else {
log.Print("new CSR for ", name)
}
sgr, err := ca.Signer(cfg.Signing)
if err != nil {
return
}
generator := &csr.Generator{Validator: func(_ *csr.CertificateRequest) error { return nil }}
csr, tlsKey, err := generator.ProcessRequest(req)
if err != nil {
return
}
signReq := signer.SignRequest{
Request: string(csr),
Profile: profile,
Label: label,
}
cert, err := sgr.Sign(signReq)
if err != nil {
return
}
kc = KeyCert{
Key: tlsKey,
Cert: cert,
ReqHash: rh,
}
err = clusterCASignedKeys.Put(key, kc)
return
}

View File

@ -0,0 +1,24 @@
package main
import (
"crypto/rand"
"encoding/base32"
"log"
"net/http"
"m.cluseau.fr/go/httperr"
)
func newToken(sizeInBytes int) (token string, err error) {
randBytes := make([]byte, sizeInBytes)
_, err = rand.Read(randBytes)
if err != nil {
log.Print("rand read error: ", err)
err = httperr.New(http.StatusInternalServerError, err)
return
}
token = base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(randBytes)
return
}

View File

@ -0,0 +1,45 @@
package main
import (
"encoding/base32"
"io"
"io/fs"
"log"
"strings"
"github.com/cespare/xxhash"
dlshtml "novit.tech/direktil/local-server/html"
)
func computeUIHash() {
xxh := xxhash.New()
err := fs.WalkDir(dlshtml.FS, "ui", func(path string, entry fs.DirEntry, walkErr error) (err error) {
if walkErr != nil {
err = walkErr
return
}
if entry.IsDir() {
return
}
f, err := dlshtml.FS.Open(path)
if err != nil {
return
}
defer f.Close()
io.Copy(xxh, f)
return nil
})
if err != nil {
log.Fatal("failed to hash UI: ", err)
}
h := strings.ToLower(base32.HexEncoding.WithPadding(base32.NoPadding).EncodeToString(xxh.Sum(nil)))[:5]
log.Printf("UI hash: %s", h)
wPublicState.Change(func(v *PublicState) { v.UIHash = h })
}

View File

@ -1,18 +1,25 @@
package main package main
import ( import (
"crypto/sha1"
"encoding/hex"
"flag" "flag"
"fmt"
"io" "io"
"log" "log"
"net/http" "net/http"
"os" "os"
gopath "path" gopath "path"
"path/filepath" "path/filepath"
"strconv"
"time" "time"
"github.com/dustin/go-humanize"
"github.com/miolini/datacounter"
) )
var ( var (
upstreamURL = flag.String("upstream", "https://direktil.novit.nc/dist", "Upstream server for dist elements") upstreamURL = flag.String("upstream", "https://dkl.novit.io/dist", "Upstream server for dist elements")
) )
func (ctx *renderContext) distFetch(path ...string) (outPath string, err error) { func (ctx *renderContext) distFetch(path ...string) (outPath string, err error) {
@ -39,42 +46,82 @@ func (ctx *renderContext) distFetch(path ...string) (outPath string, err error)
return return
} }
tempOutPath := filepath.Join(filepath.Dir(outPath), "._part_"+filepath.Base(outPath)) defer resp.Body.Close()
if resp.StatusCode != 200 {
err = fmt.Errorf("wrong status: %s", resp.Status)
return
}
length, _ := strconv.Atoi(resp.Header.Get("Content-Length"))
fOut, err := os.Create(filepath.Join(filepath.Dir(outPath), "._part_"+filepath.Base(outPath)))
if err != nil {
return
}
hash := sha1.New()
body := datacounter.NewReaderCounter(resp.Body)
out := io.MultiWriter(fOut, hash)
done := make(chan error, 1) done := make(chan error, 1)
go func() { go func() {
defer resp.Body.Close() _, err = io.Copy(out, body)
defer close(done) fOut.Close()
out, err := os.Create(tempOutPath)
if err != nil { if err != nil {
done <- err os.Remove(fOut.Name())
return
} }
defer out.Close()
_, err = io.Copy(out, resp.Body)
done <- err done <- err
close(done)
}() }()
start := time.Now()
wait: wait:
select { select {
case <-time.After(10 * time.Second): case <-time.After(10 * time.Second):
log.Print("still fetching ", subPath, "...") status := ""
if length != 0 {
count := body.Count()
elapsedDuration := time.Since(start)
progress := float64(count) / float64(length)
elapsed := float64(elapsedDuration)
remaining := time.Duration(elapsed/progress - elapsed)
status = fmt.Sprintf(" (%.2f%%, ETA %v, %s/s)",
progress*100,
remaining.Truncate(time.Second),
humanize.Bytes(uint64(float64(count)/elapsedDuration.Seconds())))
}
log.Printf("still fetching %s%s...", subPath, status)
goto wait goto wait
case err = <-done: case err = <-done:
if err != nil { if err != nil {
log.Print("fetch of ", subPath, " failed: ", err) log.Print("fetch of ", subPath, " failed: ", err)
os.Remove(tempOutPath)
return return
} }
log.Print("fetch of ", subPath, " finished")
} }
err = os.Rename(tempOutPath, outPath) hexSum := hex.EncodeToString(hash.Sum(nil))
log.Printf("fetch of %s finished (SHA1 checksum: %s)", subPath, hexSum)
if remoteSum := resp.Header.Get("X-Content-SHA1"); remoteSum != "" {
log.Printf("fetch of %s: remote SHA1 checksum: %s", subPath, remoteSum)
if remoteSum != hexSum {
err = fmt.Errorf("wrong SHA1 checksum: server=%s local=%s", remoteSum, hexSum)
log.Print("fetch of ", subPath, ": ", err)
os.Remove(fOut.Name())
return
}
}
err = os.Rename(fOut.Name(), outPath)
return return
} }

View File

@ -7,24 +7,20 @@ import (
) )
func adminAuth(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) { func adminAuth(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) {
tokenAuth(req, resp, chain, *adminToken) tokenAuth(req, resp, chain, adminToken)
}
func hostsAuth(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) {
tokenAuth(req, resp, chain, *hostsToken, *adminToken)
} }
func tokenAuth(req *restful.Request, resp *restful.Response, chain *restful.FilterChain, allowedTokens ...string) { func tokenAuth(req *restful.Request, resp *restful.Response, chain *restful.FilterChain, allowedTokens ...string) {
token := getToken(req) token := getToken(req)
for _, allowedToken := range allowedTokens { for _, allowedToken := range allowedTokens {
if allowedToken == "" || token == allowedToken { if allowedToken != "" && token == allowedToken {
chain.ProcessFilter(req, resp) chain.ProcessFilter(req, resp)
return return
} }
} }
resp.WriteErrorString(401, "401: Not Authorized") wsError(resp, ErrUnauthorized)
return return
} }
@ -33,8 +29,12 @@ func getToken(req *restful.Request) string {
token := req.HeaderParameter("Authorization") token := req.HeaderParameter("Authorization")
if token == "" {
return req.QueryParameter("token")
}
if !strings.HasPrefix(token, bearerPrefix) { if !strings.HasPrefix(token, bearerPrefix) {
return "" return token
} }
return token[len(bearerPrefix):] return token[len(bearerPrefix):]

View File

@ -0,0 +1,83 @@
package main
import (
"fmt"
"github.com/cloudflare/cfssl/log"
restful "github.com/emicklei/go-restful"
)
var clusterCAs = newClusterSecretKV[CA]("CAs")
func wsClusterCAs(req *restful.Request, resp *restful.Response) {
clusterName := req.PathParameter("cluster-name")
clusterCAs.WsList(resp, clusterName+"/")
}
func wsClusterCA(req *restful.Request, resp *restful.Response) {
clusterName := req.PathParameter("cluster-name")
name := req.PathParameter("ca-name")
clusterCAs.WsGet(resp, clusterName+"/"+name)
}
func getUsableClusterCA(cluster, name string) (ca CA, err error) {
defer func() {
if err != nil {
err = fmt.Errorf("cluster %s CA %s: %w", cluster, name, err)
}
}()
key := cluster + "/" + name
ca, found, err := clusterCAs.Get(key)
if err != nil {
return
}
if !found {
log.Info("new CA in cluster ", cluster, ": ", name)
err = ca.Init()
if err != nil {
return
}
err = clusterCAs.Put(key, ca)
if err != nil {
return
}
return
}
checkErr := checkCertUsable(ca.Cert)
if checkErr != nil {
log.Infof("cluster %s: CA %s: regenerating certificate: %v", cluster, name, checkErr)
err = ca.RenewCert()
if err != nil {
err = fmt.Errorf("renew: %w", err)
}
err = clusterCAs.Put(key, ca)
}
return
}
var clusterCASignedKeys = newClusterSecretKV[KeyCert]("CA-signed-keys")
func wsClusterCASignedKeys(req *restful.Request, resp *restful.Response) {
clusterName := req.PathParameter("cluster-name")
caName := req.PathParameter("ca-name")
clusterCASignedKeys.WsList(resp, clusterName+"/"+caName+"/")
}
func wsClusterCASignedKey(req *restful.Request, resp *restful.Response) {
clusterName := req.PathParameter("cluster-name")
caName := req.PathParameter("ca-name")
name := req.PathParameter("signed-name")
clusterCASignedKeys.WsGet(resp, clusterName+"/"+caName+"/"+name)
}

View File

@ -0,0 +1,32 @@
package main
import (
restful "github.com/emicklei/go-restful"
)
var seeds = newClusterSecretKV[[]byte]("seeds")
var clusterPasswords = newClusterSecretKV[string]("passwords")
func wsClusterPasswords(req *restful.Request, resp *restful.Response) {
clusterName := req.PathParameter("cluster-name")
clusterPasswords.WsList(resp, clusterName+"/")
}
func wsClusterPassword(req *restful.Request, resp *restful.Response) {
clusterName := req.PathParameter("cluster-name")
name := req.PathParameter("password-name")
clusterPasswords.WsGet(resp, clusterName+"/"+name)
}
func wsClusterSetPassword(req *restful.Request, resp *restful.Response) {
cluster := wsReadCluster(req, resp)
if cluster == nil {
return
}
name := req.PathParameter("password-name")
clusterPasswords.WsPut(req, resp, cluster.Name+"/"+name)
}

View File

@ -0,0 +1,43 @@
package main
import (
"crypto/rand"
"encoding/base32"
restful "github.com/emicklei/go-restful"
)
var clusterTokens = newClusterSecretKV[string]("tokens")
func getOrCreateClusterToken(cluster, name string) (token string, err error) {
key := cluster + "/" + name
token, found, err := clusterTokens.Get(key)
if err != nil || found {
return
}
b := make([]byte, 16)
_, err = rand.Read(b)
if err != nil {
return
}
token = base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(b)
err = clusterTokens.Put(key, token)
return
}
func wsClusterTokens(req *restful.Request, resp *restful.Response) {
clusterName := req.PathParameter("cluster-name")
clusterTokens.WsList(resp, clusterName+"/")
}
func wsClusterToken(req *restful.Request, resp *restful.Response) {
clusterName := req.PathParameter("cluster-name")
name := req.PathParameter("token-name")
clusterTokens.WsGet(resp, clusterName+"/"+name)
}

View File

@ -2,11 +2,22 @@ package main
import ( import (
"log" "log"
"net/url"
"strconv"
restful "github.com/emicklei/go-restful" restful "github.com/emicklei/go-restful"
"novit.nc/direktil/pkg/localconfig"
"novit.tech/direktil/local-server/pkg/mime"
"novit.tech/direktil/pkg/localconfig"
) )
var clusterSecretKVs = []string{}
func newClusterSecretKV[T any](name string) KVSecrets[T] {
clusterSecretKVs = append(clusterSecretKVs, name)
return KVSecrets[T]{"clusters/" + name}
}
func wsListClusters(req *restful.Request, resp *restful.Response) { func wsListClusters(req *restful.Request, resp *restful.Response) {
cfg := wsReadConfig(resp) cfg := wsReadConfig(resp)
if cfg == nil { if cfg == nil {
@ -31,7 +42,7 @@ func wsReadCluster(req *restful.Request, resp *restful.Response) (cluster *local
cluster = cfg.Cluster(clusterName) cluster = cfg.Cluster(clusterName)
if cluster == nil { if cluster == nil {
wsNotFound(req, resp) wsNotFound(resp)
return return
} }
@ -55,9 +66,58 @@ func wsClusterAddons(req *restful.Request, resp *restful.Response) {
if len(cluster.Addons) == 0 { if len(cluster.Addons) == 0 {
log.Printf("cluster %q has no addons defined", cluster.Name) log.Printf("cluster %q has no addons defined", cluster.Name)
wsNotFound(req, resp) wsNotFound(resp)
return return
} }
resp.Write([]byte(cluster.Addons)) cfg := wsReadConfig(resp)
if cfg == nil {
return
}
sslCfg, err := sslConfigFromLocalConfig(cfg)
if err != nil {
wsError(resp, err)
return
}
wsRender(resp, sslCfg, cluster.Addons, cluster)
}
func wsClusterCACert(req *restful.Request, resp *restful.Response) {
clusterName := req.PathParameter("cluster-name")
caName := req.PathParameter("ca-name")
ca, found, err := clusterCAs.Get(clusterName + "/" + caName)
if err != nil {
wsError(resp, err)
return
}
if !found {
wsNotFound(resp)
return
}
resp.Header().Set("Content-Type", mime.CERT)
resp.Write(ca.Cert)
}
func wsClusterSignedCert(req *restful.Request, resp *restful.Response) {
clusterName := req.PathParameter("cluster-name")
caName := req.PathParameter("ca-name")
name := req.QueryParameter("name")
kc, found, err := clusterCASignedKeys.Get(clusterName + "/" + caName + "/" + name)
if err != nil {
wsError(resp, err)
return
}
if !found {
wsNotFound(resp)
return
}
resp.AddHeader("Content-Type", mime.CERT)
resp.AddHeader("Content-Disposition", "attachment; filename="+strconv.Quote(clusterName+"_"+caName+"_"+url.PathEscape(name)+".crt"))
resp.Write(kc.Cert)
} }

View File

@ -18,7 +18,10 @@ func wsUploadConfig(req *restful.Request, resp *restful.Response) {
if err != nil { if err != nil {
wsError(resp, err) wsError(resp, err)
return
} }
resp.WriteEntity(true)
} }
func writeNewConfig(reader io.Reader) (err error) { func writeNewConfig(reader io.Reader) (err error) {
@ -35,8 +38,30 @@ func writeNewConfig(reader io.Reader) (err error) {
return return
} }
archivesPath := filepath.Join(*dataDir, "archives")
cfgPath := configFilePath() cfgPath := configFilePath()
in, err := os.Open(cfgPath)
if err != nil {
if os.IsNotExist(err) {
// nothing to backup
} else {
return // real error
}
} else {
err = backupCurrentConfig(in)
if err != nil {
return
}
}
err = os.Rename(out.Name(), cfgPath)
updateState()
return
}
func backupCurrentConfig(in io.ReadCloser) (err error) {
archivesPath := filepath.Join(*dataDir, "archives")
err = os.MkdirAll(archivesPath, 0700) err = os.MkdirAll(archivesPath, 0700)
if err != nil { if err != nil {
@ -52,11 +77,6 @@ func writeNewConfig(reader io.Reader) (err error) {
defer bck.Close() defer bck.Close()
in, err := os.Open(cfgPath)
if err != nil {
return
}
gz, err := gzip.NewWriterLevel(bck, 2) gz, err := gzip.NewWriterLevel(bck, 2)
if err != nil { if err != nil {
return return
@ -66,10 +86,5 @@ func writeNewConfig(reader io.Reader) (err error) {
gz.Close() gz.Close()
in.Close() in.Close()
if err != nil {
return
}
err = os.Rename(out.Name(), cfgPath)
return return
} }

View File

@ -0,0 +1,151 @@
package main
import (
"crypto/rand"
"encoding/base32"
"log"
"net/http"
"strconv"
"time"
restful "github.com/emicklei/go-restful"
"m.cluseau.fr/go/cow"
)
type DownloadSpec struct {
Kind string
Name string
Assets []string
createdAt time.Time
}
func wsAuthorizeDownload(req *restful.Request, resp *restful.Response) {
var spec DownloadSpec
if err := req.ReadEntity(&spec); err != nil {
wsError(resp, err)
return
}
if spec.Kind == "" || spec.Name == "" || len(spec.Assets) == 0 {
resp.WriteErrorString(http.StatusBadRequest, "missing data")
return
}
randBytes := make([]byte, 32)
_, err := rand.Read(randBytes)
if err != nil {
wsError(resp, err)
return
}
token := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(randBytes)
spec.createdAt = time.Now()
wState.Change(func(v *State) {
cow.MapSet(&v.Downloads, token, spec)
})
log.Printf("download token created for %s %q, assets %q", spec.Kind, spec.Name, spec.Assets)
resp.WriteAsJson(token)
}
func wsDownload(req *restful.Request, resp *restful.Response) {
token := req.PathParameter("token")
asset := req.PathParameter("asset")
if token == "" || asset == "" {
wsNotFound(resp)
return
}
var spec DownloadSpec
found := false
wState.Change(func(v *State) {
var ok bool
spec, ok = v.Downloads[token]
if !ok {
return
}
newAssets := make([]string, 0, len(spec.Assets))
for _, a := range spec.Assets {
if a == asset {
found = true
} else {
newAssets = append(newAssets, a)
}
}
if !found {
wsNotFound(resp)
return
}
cow.Map(&v.Downloads)
if len(newAssets) == 0 {
delete(v.Downloads, token)
} else {
spec.Assets = newAssets
v.Downloads[token] = spec
}
})
if !found {
wsNotFound(resp)
return
}
log.Printf("download via token: %s %q asset %q", spec.Kind, spec.Name, asset)
cfg, err := readConfig()
if err != nil {
wsError(resp, err)
return
}
setHeader := func(ext string) {
resp.AddHeader("Content-Disposition", "attachment; filename="+strconv.Quote(spec.Kind+"_"+spec.Name+"_"+asset+ext))
}
switch spec.Kind {
case "cluster":
cluster := cfg.ClusterByName(spec.Name)
if cluster == nil {
wsNotFound(resp)
return
}
switch asset {
case "addons":
setHeader(".yaml")
resp.Write([]byte(cluster.Addons))
default:
wsNotFound(resp)
}
case "host":
host := cfg.Host(spec.Name)
if host == nil {
wsNotFound(resp)
return
}
switch asset {
case "config", "bootstrap-config":
setHeader(".yaml")
default:
setHeader("")
}
renderHost(resp.ResponseWriter, req.Request, asset, host, cfg)
default:
wsNotFound(resp)
}
}

View File

@ -8,26 +8,30 @@ import (
restful "github.com/emicklei/go-restful" restful "github.com/emicklei/go-restful"
"novit.nc/direktil/local-server/pkg/mime" "novit.tech/direktil/pkg/localconfig"
"novit.nc/direktil/pkg/localconfig"
"novit.tech/direktil/local-server/pkg/mime"
) )
var trustXFF = flag.Bool("trust-xff", true, "Trust the X-Forwarded-For header") var (
allowDetectedHost = flag.Bool("allow-detected-host", false, "Allow access to host assets from its IP (insecure but enables unattended netboot)")
trustXFF = flag.Bool("trust-xff", false, "Trust the X-Forwarded-For header")
)
type wsHost struct { type wsHost struct {
prefix string
hostDoc string hostDoc string
getHost func(req *restful.Request) string getHost func(req *restful.Request) (hostName string, err error)
} }
func (ws *wsHost) register(rws *restful.WebService, alterRB func(*restful.RouteBuilder)) { func (ws wsHost) register(rws *restful.WebService, alterRB func(*restful.RouteBuilder)) {
b := func(what string) *restful.RouteBuilder { b := func(what string) *restful.RouteBuilder {
return rws.GET(ws.prefix + "/" + what).To(ws.render) return rws.GET("/" + what).To(ws.render)
} }
for _, rb := range []*restful.RouteBuilder{ for _, rb := range []*restful.RouteBuilder{
rws.GET(ws.prefix).To(ws.get). rws.GET("").To(ws.get).
Doc("Get the " + ws.hostDoc + "'s details"), Doc("Get the "+ws.hostDoc+"'s details").
Returns(200, "OK", localconfig.Host{}),
// raw configuration // raw configuration
b("config"). b("config").
@ -54,10 +58,14 @@ func (ws *wsHost) register(rws *restful.WebService, alterRB func(*restful.RouteB
b("boot.tar"). b("boot.tar").
Produces(mime.TAR). Produces(mime.TAR).
Doc("Get the " + ws.hostDoc + "'s /boot archive (ie: for metal upgrades)"), Doc("Get the " + ws.hostDoc + "'s /boot archive (ie: for metal upgrades)"),
b("boot-efi.tar").
Produces(mime.TAR).
Doc("Get the " + ws.hostDoc + "'s /boot archive (ie: for metal upgrades)"),
// read-only ISO support // read-only ISO support
b("boot.iso"). b("boot.iso").
Produces(mime.ISO). Produces(mime.ISO).
Param(cmdlineParam).
Doc("Get the " + ws.hostDoc + "'s boot CD-ROM image"), Doc("Get the " + ws.hostDoc + "'s boot CD-ROM image"),
// netboot support // netboot support
@ -65,27 +73,42 @@ func (ws *wsHost) register(rws *restful.WebService, alterRB func(*restful.RouteB
Produces(mime.IPXE). Produces(mime.IPXE).
Doc("Get the " + ws.hostDoc + "'s IPXE code (for netboot)"), Doc("Get the " + ws.hostDoc + "'s IPXE code (for netboot)"),
// boot support
b("kernel"). b("kernel").
Produces(mime.OCTET). Produces(mime.OCTET).
Doc("Get the " + ws.hostDoc + "'s kernel (ie: for netboot)"), Doc("Get the " + ws.hostDoc + "'s kernel (ie: for netboot)"),
b("initrd"). b("initrd").
Produces(mime.OCTET). Produces(mime.OCTET).
Doc("Get the " + ws.hostDoc + "'s initial RAM disk (ie: for netboot)"), Doc("Get the " + ws.hostDoc + "'s initial RAM disk (ie: for netboot)"),
// - bootstrap config
b("bootstrap-config").
Produces(mime.YAML).
Doc("Get the " + ws.hostDoc + "'s bootstrap configuration"),
b("bootstrap-config.json").
Doc("Get the " + ws.hostDoc + "'s bootstrap configuration (as JSON)"),
// - bootstrap
b("bootstrap.tar").
Produces(mime.TAR).
Doc("Get the " + ws.hostDoc + "'s bootstrap seed archive"),
} { } {
alterRB(rb) alterRB(rb)
rws.Route(rb) rws.Route(rb)
} }
} }
func (ws *wsHost) host(req *restful.Request, resp *restful.Response) (host *localconfig.Host, cfg *localconfig.Config) { func (ws wsHost) host(req *restful.Request, resp *restful.Response) (host *localconfig.Host, cfg *localconfig.Config) {
hostname := ws.getHost(req) hostname, err := ws.getHost(req)
if err != nil {
wsError(resp, err)
return
}
if hostname == "" { if hostname == "" {
wsNotFound(req, resp) wsNotFound(resp)
return return
} }
cfg, err := readConfig() cfg, err = readConfig()
if err != nil { if err != nil {
wsError(resp, err) wsError(resp, err)
return return
@ -94,13 +117,13 @@ func (ws *wsHost) host(req *restful.Request, resp *restful.Response) (host *loca
host = cfg.Host(hostname) host = cfg.Host(hostname)
if host == nil { if host == nil {
log.Print("no host named ", hostname) log.Print("no host named ", hostname)
wsNotFound(req, resp) wsNotFound(resp)
return return
} }
return return
} }
func (ws *wsHost) get(req *restful.Request, resp *restful.Response) { func (ws wsHost) get(req *restful.Request, resp *restful.Response) {
host, _ := ws.host(req, resp) host, _ := ws.host(req, resp)
if host == nil { if host == nil {
return return
@ -109,7 +132,7 @@ func (ws *wsHost) get(req *restful.Request, resp *restful.Response) {
resp.WriteEntity(host) resp.WriteEntity(host)
} }
func (ws *wsHost) render(req *restful.Request, resp *restful.Response) { func (ws wsHost) render(req *restful.Request, resp *restful.Response) {
host, cfg := ws.host(req, resp) host, cfg := ws.host(req, resp)
if host == nil { if host == nil {
return return
@ -141,21 +164,26 @@ func renderHost(w http.ResponseWriter, r *http.Request, what string, host *local
case "kernel": case "kernel":
err = renderKernel(w, r, ctx) err = renderKernel(w, r, ctx)
// boot v2
case "bootstrap-config":
err = renderBootstrapConfig(w, r, ctx, false)
case "bootstrap-config.json":
err = renderBootstrapConfig(w, r, ctx, true)
case "initrd": case "initrd":
err = renderCtx(w, r, ctx, what, buildInitrd) err = renderCtx(w, r, ctx, what, buildInitrd)
case "bootstrap.tar":
err = renderCtx(w, r, ctx, what, buildBootstrap)
case "boot.iso": case "boot.iso":
err = renderCtx(w, r, ctx, what, buildBootISO) err = renderCtx(w, r, ctx, what, buildBootISO)
case "boot.tar": case "boot.tar":
err = renderCtx(w, r, ctx, what, buildBootTar) err = renderCtx(w, r, ctx, what, buildBootTar)
case "boot-efi.tar":
err = renderCtx(w, r, ctx, what, buildBootEFITar)
case "boot.img": case "boot.img":
err = renderCtx(w, r, ctx, what, buildBootImg) err = renderCtx(w, r, ctx, what, buildBootImg)
case "boot.img.gz": case "boot.img.gz":
err = renderCtx(w, r, ctx, what, buildBootImgGZ) err = renderCtx(w, r, ctx, what, buildBootImgGZ)
case "boot.img.lz4": case "boot.img.lz4":
err = renderCtx(w, r, ctx, what, buildBootImgLZ4) err = renderCtx(w, r, ctx, what, buildBootImgLZ4)

View File

@ -0,0 +1,195 @@
package main
import (
"archive/tar"
"bytes"
"io"
"io/fs"
"log"
"net/http"
"os"
"path/filepath"
restful "github.com/emicklei/go-restful"
"m.cluseau.fr/go/httperr"
"novit.tech/direktil/local-server/secretstore"
)
type NamedPassphrase struct {
Name string
Passphrase []byte
}
func wsUnlockStore(req *restful.Request, resp *restful.Response) {
np := NamedPassphrase{}
err := req.ReadEntity(&np)
if err != nil {
resp.WriteError(http.StatusBadRequest, err)
return
}
defer secretstore.Memzero(np.Passphrase)
if secStore.IsNew() {
if len(np.Name) == 0 {
wsBadRequest(resp, "no name given")
return
}
}
if len(np.Passphrase) == 0 {
wsBadRequest(resp, "no passphrase given")
return
}
if secStore.Unlocked() {
if secStore.HasKey(np.Passphrase) {
resp.WriteEntity(adminToken)
} else {
wsError(resp, ErrUnauthorized)
}
return
}
if err := unlockSecretStore(np.Name, np.Passphrase); err.Any() {
err.WriteJSON(resp.ResponseWriter)
return
}
resp.WriteEntity(adminToken)
}
func wsStoreDownload(req *restful.Request, resp *restful.Response) {
token := req.QueryParameter("token")
if token != wState.Get().Store.DownloadToken {
wsError(resp, ErrInvalidToken)
return
}
buf := new(bytes.Buffer)
arch := tar.NewWriter(buf)
root := os.DirFS(secStoreRoot())
err := fs.WalkDir(root, ".", func(path string, d fs.DirEntry, readErr error) (err error) {
if readErr != nil {
err = readErr
return
}
if path == "." {
return
}
fi, err := d.Info()
if err != nil {
return
}
hdr, err := tar.FileInfoHeader(fi, "")
if err != nil {
return
}
hdr.Name = path
hdr.Uid = 0
hdr.Gid = 0
err = arch.WriteHeader(hdr)
if err != nil {
return
}
if fi.IsDir() {
return
}
f, err := root.Open(path)
if err != nil {
return
}
defer f.Close()
io.Copy(arch, f)
return
})
if err != nil {
wsError(resp, err)
return
}
err = arch.Close()
if err != nil {
wsError(resp, err)
return
}
buf.WriteTo(resp)
}
func wsStoreUpload(req *restful.Request, resp *restful.Response) {
if !secStore.IsNew() {
wsError(resp, httperr.BadRequest("store is not new"))
return
}
buf := new(bytes.Buffer)
_, err := io.Copy(buf, req.Request.Body)
if err != nil {
wsError(resp, err)
return
}
arch := tar.NewReader(buf)
root := secStoreRoot()
for {
hdr, err := arch.Next()
if err == io.EOF {
err = nil
break
} else if err != nil {
wsError(resp, err)
return
}
log.Print(hdr.Name)
fullPath := filepath.Join(root, hdr.Name)
switch {
case hdr.FileInfo().IsDir():
err = os.MkdirAll(fullPath, 0700)
if err != nil {
wsError(resp, err)
return
}
default:
content, err := io.ReadAll(io.LimitReader(arch, hdr.Size))
if err != nil {
wsError(resp, err)
return
}
err = os.WriteFile(fullPath, content, 0600)
if err != nil {
wsError(resp, err)
return
}
}
}
if err != nil {
wsError(resp, err)
return
}
openSecretStore()
resp.WriteEntity(map[string]any{"ok": true})
}

View File

@ -0,0 +1,44 @@
package main
import (
"net/http"
"os"
"path/filepath"
restful "github.com/emicklei/go-restful"
yaml "gopkg.in/yaml.v2"
)
type SSH_ACL struct {
Keys []string
Clusters []string
Groups []string
Hosts []string
}
func loadSSH_ACLs() (acls []SSH_ACL, err error) {
f, err := os.Open(filepath.Join(*dataDir, "ssh-acls.yaml"))
if err != nil {
return
}
defer f.Close()
err = yaml.NewDecoder(f).Decode(&acls)
return
}
func wsSSH_ACL_List(req *restful.Request, resp *restful.Response) {
// TODO
http.NotFound(resp.ResponseWriter, req.Request)
}
func wsSSH_ACL_Get(req *restful.Request, resp *restful.Response) {
// TODO
http.NotFound(resp.ResponseWriter, req.Request)
}
func wsSSH_ACL_Set(req *restful.Request, resp *restful.Response) {
// TODO
http.NotFound(resp.ResponseWriter, req.Request)
}

View File

@ -0,0 +1,80 @@
package main
import (
"strconv"
"strings"
restful "github.com/emicklei/go-restful"
"novit.tech/direktil/local-server/secretstore"
)
func wsStoreAddKey(req *restful.Request, resp *restful.Response) {
np := NamedPassphrase{}
err := req.ReadEntity(&np)
if err != nil {
wsBadRequest(resp, err.Error())
return
}
np.Name = strings.TrimSpace(np.Name)
if len(np.Name) == 0 {
wsBadRequest(resp, "no name given")
return
}
if len(np.Passphrase) == 0 {
wsBadRequest(resp, "no passphrase given")
return
}
secStore.AddKey(np.Name, np.Passphrase)
defer updateState()
for _, k := range secStore.Keys {
if k.Name == np.Name {
wsBadRequest(resp, "there's already a passphrase named "+strconv.Quote(np.Name))
return
}
}
err = secStore.SaveTo(secKeysStorePath())
if err != nil {
wsError(resp, err)
return
}
}
func wsStoreDelKey(req *restful.Request, resp *restful.Response) {
name := ""
err := req.ReadEntity(&name)
if err != nil {
wsBadRequest(resp, err.Error())
return
}
newKeys := make([]secretstore.KeyEntry, 0, len(secStore.Keys))
for _, k := range secStore.Keys {
if k.Name == name {
continue
}
newKeys = append(newKeys, k)
}
if len(newKeys) == 0 {
wsBadRequest(resp, "can't remove the last key from the store")
return
}
secStore.Keys = newKeys
defer updateState()
err = secStore.SaveTo(secKeysStorePath())
if err != nil {
wsError(resp, err)
return
}
}

View File

@ -1,62 +1,211 @@
package main package main
import ( import (
"errors"
"fmt"
"log" "log"
"net" "net"
"net/http" "net/http"
"strings" "strings"
"text/template"
cfsslconfig "github.com/cloudflare/cfssl/config"
"github.com/emicklei/go-restful" "github.com/emicklei/go-restful"
"novit.nc/direktil/local-server/pkg/mime" "m.cluseau.fr/go/httperr"
"novit.nc/direktil/pkg/localconfig"
"novit.tech/direktil/pkg/localconfig"
"novit.tech/direktil/local-server/pkg/mime"
) )
func buildWS() *restful.WebService { func registerWS(rest *restful.Container) {
ws := &restful.WebService{} // public-level APIs
{
ws := &restful.WebService{}
ws.
Path("/public").
Produces(mime.JSON).
Consumes(mime.JSON).
Route(ws.POST("/unlock-store").To(wsUnlockStore).
Reads(NamedPassphrase{}).
Writes("").
Doc("Try to unlock the store")).
Route(ws.GET("/store.tar").To(wsStoreDownload).
Produces(mime.TAR).
Param(ws.QueryParameter("token", "the download token")).
Doc("Fetch the encrypted store")).
Route(ws.POST("/store.tar").To(wsStoreUpload).
Consumes(mime.TAR).
Doc("Upload an existing store")).
Route(ws.GET("/downloads/{token}/{asset}").To(wsDownload).
Param(ws.PathParameter("token", "the download token")).
Param(ws.PathParameter("asset", "the requested asset")).
Doc("Fetch an asset via a download token"))
// configs API rest.Add(ws)
ws.Route(ws.POST("/configs").Filter(adminAuth).To(wsUploadConfig). }
// Admin-level APIs
ws := (&restful.WebService{}).
Filter(requireSecStore).
Filter(adminAuth).
Param(restful.HeaderParameter("Authorization", "Admin bearer token").Required(true)).
Produces(mime.JSON)
// - store management
ws.Route(ws.POST("/store/add-key").To(wsStoreAddKey).
Consumes(mime.JSON).Reads(NamedPassphrase{}).
Doc("Add an unlock key to the store"))
ws.Route(ws.POST("/store/delete-key").To(wsStoreDelKey).
Consumes(mime.JSON).Reads("").
Doc("Remove an unlock key to the store (by its name)"))
// - downloads
ws.Route(ws.POST("/authorize-download").To(wsAuthorizeDownload).
Consumes(mime.JSON).Reads(DownloadSpec{}).
Produces(mime.JSON).
Doc("Create a download token for the given download"))
// - configs API
ws.Route(ws.POST("/configs").To(wsUploadConfig).
Consumes(mime.YAML).Param(ws.BodyParameter("config", "The new full configuration")).
Produces(mime.JSON).Writes(true).
Doc("Upload a new current configuration, archiving the previous one")) Doc("Upload a new current configuration, archiving the previous one"))
// clusters API // - clusters API
ws.Route(ws.GET("/clusters").Filter(adminAuth).To(wsListClusters). ws.Route(ws.GET("/clusters").To(wsListClusters).
Doc("List clusters")) Doc("List clusters"))
ws.Route(ws.GET("/clusters/{cluster-name}").Filter(adminAuth).To(wsCluster). const (
Doc("Get cluster details")) GET = http.MethodGet
PUT = http.MethodPut
)
ws.Route(ws.GET("/clusters/{cluster-name}/addons").Filter(adminAuth).To(wsClusterAddons). cluster := func(method, subPath string) *restful.RouteBuilder {
Produces(mime.YAML). return ws.Method(method).Path("/clusters/{cluster-name}" + subPath).
Doc("Get cluster addons"). Param(ws.PathParameter("cluster-name", "name of the cluster"))
Returns(http.StatusOK, "OK", nil). }
Returns(http.StatusNotFound, "The cluster does not exists or does not have addons defined", nil))
// hosts API for _, builder := range []*restful.RouteBuilder{
ws.Route(ws.GET("/hosts").Filter(hostsAuth).To(wsListHosts). cluster(GET, "").To(wsCluster).
Doc("Get cluster details"),
cluster(GET, "/addons").To(wsClusterAddons).
Produces(mime.YAML).
Doc("Get cluster addons").
Returns(http.StatusOK, "OK", nil).
Returns(http.StatusNotFound, "The cluster does not exists or does not have addons defined", nil),
cluster(GET, "/tokens").To(wsClusterTokens).
Doc("List cluster's tokens"),
cluster(GET, "/tokens/{token-name}").To(wsClusterToken).
Doc("Get cluster's token"),
cluster(GET, "/passwords").To(wsClusterPasswords).
Doc("List cluster's passwords"),
cluster(GET, "/passwords/{password-name}").To(wsClusterPassword).
Doc("Get cluster's password"),
cluster(PUT, "/passwords/{password-name}").To(wsClusterSetPassword).
Doc("Set cluster's password"),
cluster(GET, "/CAs").To(wsClusterCAs).
Doc("Get cluster CAs"),
cluster(GET, "/CAs/{ca-name}/certificate").To(wsClusterCACert).
Produces(mime.CACERT).
Doc("Get cluster CA's certificate"),
cluster(GET, "/CAs/{ca-name}/signed").To(wsClusterSignedCert).
Produces(mime.CERT).
Param(ws.QueryParameter("name", "signed reference name").Required(true)).
Doc("Get cluster's certificate signed by the CA"),
} {
ws.Route(builder)
}
ws.Route(ws.GET("/hosts").To(wsListHosts).
Doc("List hosts")) Doc("List hosts"))
ws.Route(ws.GET("/ssh-acls").To(wsSSH_ACL_List))
ws.Route(ws.GET("/ssh-acls/{acl-name}").To(wsSSH_ACL_Get))
ws.Route(ws.PUT("/ssh-acls/{acl-name}").To(wsSSH_ACL_Set))
rest.Add(ws)
// Hosts API
ws = (&restful.WebService{}).
Filter(requireSecStore).
Filter(adminAuth).
Path("/hosts/{host-name}").
Param(ws.HeaderParameter("Authorization", "Host or admin bearer token"))
(&wsHost{
hostDoc: "given host",
getHost: func(req *restful.Request) (string, error) {
return req.PathParameter("host-name"), nil
},
}).register(ws, func(rb *restful.RouteBuilder) {
rb.Param(ws.PathParameter("host-name", "host's name"))
})
rest.Add(ws)
// Detected host API
ws = (&restful.WebService{}).
Filter(requireSecStore).
Path("/me").
Param(ws.HeaderParameter("Authorization", "Host or admin bearer token"))
(&wsHost{ (&wsHost{
prefix: "/me",
hostDoc: "detected host", hostDoc: "detected host",
getHost: detectHost, getHost: detectHost,
}).register(ws, func(rb *restful.RouteBuilder) { }).register(ws, func(rb *restful.RouteBuilder) {
rb.Notes("In this case, the host is detected from the remote IP") rb.Notes("In this case, the host is detected from the remote IP")
}) })
// Hosts by token API
ws = (&restful.WebService{}).
Filter(requireSecStore).
Path("/hosts-by-token/{host-token}").
Param(ws.PathParameter("host-token", "host's download token"))
(&wsHost{ (&wsHost{
prefix: "/hosts/{host-name}", hostDoc: "token's host",
hostDoc: "given host", getHost: func(req *restful.Request) (host string, err error) {
getHost: func(req *restful.Request) string { reqToken := req.PathParameter("host-token")
return req.PathParameter("host-name")
data, err := hostDownloadTokens.Data()
if err != nil {
return
}
for h, token := range data {
if token == reqToken {
host = h
return
}
}
return
}, },
}).register(ws, func(rb *restful.RouteBuilder) { }).register(ws, func(rb *restful.RouteBuilder) {
rb.Filter(adminAuth) rb.Notes("In this case, the host is detected from the token")
}) })
return ws rest.Add(ws)
} }
func detectHost(req *restful.Request) string { func requireSecStore(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) {
if !secStore.Unlocked() {
wsError(resp, ErrStoreLocked)
return
}
chain.ProcessFilter(req, resp)
}
func detectHost(req *restful.Request) (hostName string, err error) {
if !*allowDetectedHost {
return
}
r := req.Request r := req.Request
remoteAddr := r.RemoteAddr remoteAddr := r.RemoteAddr
@ -74,17 +223,17 @@ func detectHost(req *restful.Request) string {
cfg, err := readConfig() cfg, err := readConfig()
if err != nil { if err != nil {
return "" return
} }
host := cfg.HostByIP(hostIP) host := cfg.HostByIP(hostIP)
if host == nil { if host == nil {
log.Print("no host found for IP ", hostIP) log.Print("no host found for IP ", hostIP)
return "" return
} }
return host.Name return host.Name, nil
} }
func wsReadConfig(resp *restful.Response) *localconfig.Config { func wsReadConfig(resp *restful.Response) *localconfig.Config {
@ -98,13 +247,36 @@ func wsReadConfig(resp *restful.Response) *localconfig.Config {
return cfg return cfg
} }
func wsNotFound(req *restful.Request, resp *restful.Response) { func wsNotFound(resp *restful.Response) {
http.NotFound(resp.ResponseWriter, req.Request) wsError(resp, ErrNotFound)
}
func wsBadRequest(resp *restful.Response, err string) {
httperr.New(http.StatusBadRequest, errors.New(err)).WriteJSON(resp.ResponseWriter)
} }
func wsError(resp *restful.Response, err error) { func wsError(resp *restful.Response, err error) {
log.Print("request failed: ", err) log.Output(2, fmt.Sprint("request failed: ", err))
resp.WriteErrorString(
http.StatusInternalServerError, switch err := err.(type) {
http.StatusText(http.StatusInternalServerError)) case httperr.Error:
err.WriteJSON(resp.ResponseWriter)
default:
httperr.Internal(err).WriteJSON(resp.ResponseWriter)
}
}
func wsRender(resp *restful.Response, sslCfg *cfsslconfig.Config, tmplStr string, value interface{}) {
tmpl, err := template.New("wsRender").Funcs(templateFuncs(sslCfg)).Parse(tmplStr)
if err != nil {
wsError(resp, err)
return
}
err = tmpl.Execute(resp, value)
if err != nil {
wsError(resp, err)
return
}
} }

1
gen-api-js.sh Executable file
View File

@ -0,0 +1 @@
docker run --rm --net=host --user $(id -u) -v ${PWD}:/local swaggerapi/swagger-codegen-cli generate -i http://[::1]:7606/swagger.json -l javascript -o /local/js/api/

107
go.mod
View File

@ -1,34 +1,81 @@
module novit.nc/direktil/local-server module novit.tech/direktil/local-server
go 1.21
require ( require (
github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e github.com/cespare/xxhash v1.1.0
github.com/cloudflare/cfssl v0.0.0-20181213083726-b94e044bb51e github.com/cloudflare/cfssl v1.6.4
github.com/coreos/etcd v3.3.11+incompatible // indirect github.com/dustin/go-humanize v1.0.1
github.com/emicklei/go-restful v2.8.1+incompatible github.com/emicklei/go-restful v2.16.0+incompatible
github.com/emicklei/go-restful-openapi v1.0.0 github.com/emicklei/go-restful-openapi v1.4.1
github.com/go-openapi/jsonpointer v0.18.0 // indirect github.com/go-git/go-git/v5 v5.10.0
github.com/go-openapi/jsonreference v0.18.0 // indirect github.com/mcluseau/go-swagger-ui v0.0.0-20191019002626-fd9128c24a34
github.com/go-openapi/spec v0.18.0 // indirect github.com/miolini/datacounter v1.0.3
github.com/go-openapi/swag v0.18.0 // indirect
github.com/gobuffalo/buffalo-plugins v1.12.1 // indirect
github.com/google/certificate-transparency-go v1.0.21 // indirect
github.com/json-iterator/go v1.1.5 // indirect
github.com/markbates/going v1.0.3 // indirect
github.com/mcluseau/go-swagger-ui v0.0.0-20190204031235-fc4ac9154422
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/oklog/ulid v1.3.1 github.com/oklog/ulid v1.3.1
github.com/pierrec/lz4 v2.0.5+incompatible github.com/pierrec/lz4 v2.6.1+incompatible
github.com/spf13/afero v1.2.1 // indirect github.com/sergeymakinen/go-crypt v1.0.0-beta.0
github.com/src-d/go-git v4.7.0+incompatible // indirect golang.org/x/crypto v0.14.0
github.com/ugorji/go/codec v0.0.0-20190128213124-ee1426cffec0 // indirect gopkg.in/src-d/go-billy.v4 v4.3.2
golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 // indirect gopkg.in/src-d/go-git.v4 v4.13.1
golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3 // indirect gopkg.in/yaml.v2 v2.4.0
golang.org/x/sys v0.0.0-20190203050204-7ae0202eb74c // indirect k8s.io/apimachinery v0.28.3
golang.org/x/tools v0.0.0-20190202235157-7414d4c1f71c // indirect m.cluseau.fr/go v0.0.0-20230809064045-12c5a121c766
gopkg.in/src-d/go-billy.v4 v4.3.0 novit.tech/direktil/pkg v0.0.0-20230201224712-5e39572dc50e
gopkg.in/src-d/go-git.v4 v4.10.0 )
gopkg.in/yaml.v2 v2.2.2
k8s.io/apimachinery v0.0.0-20190201131811-df262fa1a1ba replace github.com/zmap/zlint/v3 => github.com/zmap/zlint/v3 v3.3.1
novit.nc/direktil/pkg v0.0.0-20181210211743-9dc80cd34b09
require (
dario.cat/mergo v1.0.0 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c // indirect
github.com/acomagu/bufpipe v1.0.4 // indirect
github.com/cavaliergopher/cpio v1.0.1 // indirect
github.com/cloudflare/circl v1.3.6 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/frankban/quicktest v1.5.0 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.5.0 // indirect
github.com/go-logr/logr v1.3.0 // indirect
github.com/go-openapi/jsonpointer v0.20.0 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/spec v0.20.9 // indirect
github.com/go-openapi/swag v0.22.4 // indirect
github.com/gobuffalo/envy v1.10.2 // indirect
github.com/gobuffalo/packd v1.0.2 // indirect
github.com/gobuffalo/packr v1.30.1 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/certificate-transparency-go v1.1.7 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/jmoiron/sqlx v1.3.5 // indirect
github.com/joho/godotenv v1.5.1 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/kisielk/sqlstruct v0.0.0-20210630145711-dae28ed37023 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/rogpeppe/go-internal v1.11.0 // indirect
github.com/sergi/go-diff v1.3.1 // indirect
github.com/skeema/knownhosts v1.2.1 // indirect
github.com/src-d/gcfg v1.4.0 // indirect
github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/zmap/zcrypto v0.0.0-20231102161736-a55ea7b96dbc // indirect
github.com/zmap/zlint/v3 v3.1.0 // indirect
golang.org/x/mod v0.13.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/tools v0.14.0 // indirect
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/klog/v2 v2.110.1 // indirect
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
) )

993
go.sum

File diff suppressed because it is too large Load Diff

3
hack/build Executable file
View File

@ -0,0 +1,3 @@
#! /bin/sh
set -ex
go build -o dist/ -trimpath -ldflags "-X main.Version=${GIT_TAG:-$(git describe --always --dirty)}" $*

8
hack/docker-build Executable file
View File

@ -0,0 +1,8 @@
#! /bin/bash
set -ex
case "$1" in
commit) tag=$(git describe --always --dirty) ;;
"") tag=latest ;;
*) tag=$1 ;;
esac
docker build -t novit.tech/direktil/local-server:$tag .

4
hack/install Executable file
View File

@ -0,0 +1,4 @@
#! /bin/sh
set -ex
go install -trimpath -ldflags "-X main.Version=$(git describe --always --dirty)" \
./cmd/dkl-dir2config

BIN
html/favicon.ico Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.7 KiB

6
html/html.go Normal file
View File

@ -0,0 +1,6 @@
package dlshtml
import "embed"
//go:embed favicon.ico ui
var FS embed.FS

23
html/ui/app.css Normal file
View File

@ -0,0 +1,23 @@
.downloads {
display: flex;
align-content: stretch;
}
.downloads > * {
margin-left: 6pt;
}
.downloads > *:first-child {
margin-left: 0;
}
.downloads > div {
display: flex;
flex-flow: column;
max-height: 100pt;
overflow: auto;
}
.cluster {
max-width: 50%;
}

116
html/ui/index.html Normal file
View File

@ -0,0 +1,116 @@
<!doctype html>
<html>
<head>
<title>Direktil Local Server</title>
<style>
@import url('./style.css');
@import url('./app.css');
</style>
<script src="js/jsonpatch.min.js" crossorigin="anonymous"></script>
<script src="js/app.js" type="module" defer></script>
<body>
<div id="app">
<header>
<div id="logo">
<img src="/favicon.ico" />
<span>Direktil Local Server</span>
</div>
<div class="utils">
<span id="login-hdr" v-if="session.token">
Logged in
<button class="link" @click="copyText(session.token)">&#x1F5D0;</button>
</span>
<span>server <code>{{ serverVersion || '-----' }}</code></span>
<span>ui <code>{{ uiHash || '-----' }}</code></span>
<span :class="publicState ? 'green' : 'red'">&#x1F5F2;</span>
</div>
</header>
<div class="error" v-if="error">
<button class="btn-close" @click="error=null">&times;</button>
<div class="code" v-if="error.code">{{ error.code }}</div>
<div class="message">{{ error.message }}</div>
</div>
<template v-if="!publicState">
<p>Not connected.</p>
</template>
<template v-else-if="publicState.Store.New">
<p>Store is new.</p>
<p>Option 1: initialize a new store</p>
<form @submit="unlockStore">
<input type="text" v-model="forms.store.name" name="name" placeholder="Name" /><br/>
<input type="password" v-model="forms.store.pass1" name="passphrase" required placeholder="Passphrase" />
<input type="password" v-model="forms.store.pass2" required placeholder="Passphrase confirmation" />
<input type="submit" value="initialize" :disabled="!forms.store.pass1 || forms.store.pass1 != forms.store.pass2" />
</form>
<p>Option 2: upload a previously downloaded store</p>
<form @submit="uploadStore">
<input type="file" ref="storeUpload" />
<input type="submit" value="upload" />
</form>
</template>
<template v-else-if="!publicState.Store.Open">
<p>Store is not open.</p>
<form @submit="unlockStore">
<input type="password" name="passphrase" v-model="forms.store.pass1" required placeholder="Passphrase" />
<input type="submit" value="unlock" :disabled="!forms.store.pass1" />
</form>
</template>
<template v-else-if="!state">
<p v-if="!session.token">Not logged in.</p>
<p v-else>Invalid token</p>
<form @submit="setToken">
<input type="password" v-model="forms.setToken" required placeholder="Token" />
<input type="submit" value="set token"/>
</form>
</template>
<template v-else>
<div v-if="state.Clusters" id="clusters">
<h2>Clusters</h2>
<div class="sheets">
<Cluster v-for="c in state.Clusters" :cluster="c" :token="session.token" :state="state" />
</div>
</div>
<div v-if="state.Hosts" id="hosts">
<h2>Hosts</h2>
<div class="sheets">
<Host v-for="h in state.Hosts" :host="h" :token="session.token" :state="state" />
</div>
</div>
<h2>Admin actions</h2>
<h3>Config</h3>
<form @submit="uploadConfig">
<input type="file" ref="configUpload" required />
<input type="submit" value="upload config" />
</form>
<h3>Store</h3>
<p><a :href="'/public/store.tar?token='+state.Store.DownloadToken" target="_blank">Download</a></p>
<form @submit="storeAddKey" action="/store/add-key">
<p>Add an unlock phrase:</p>
<input type="text" v-model="forms.store.name" name="name" required placeholder="Name" /><br/>
<input type="password" v-model="forms.store.pass1" name="passphrase" autocomplete="new-password" required placeholder="Phrase" />
<input type="password" v-model="forms.store.pass2" autocomplete="new-password" required placeholder="Phrase confirmation" />
<input type="submit" value="add unlock phrase" :disabled="!forms.store.pass1 || forms.store.pass1 != forms.store.pass2" />
</form>
<form @submit="storeDelKey" action="/store/delete-key">
<p>Remove an unlock phrase:</p>
<input type="text" v-model="forms.delKey.name" name="name" required placeholder="Name" />
<input type="submit" value="remove unlock phrase" />
<p v-if="state.Store.KeyNames">Available names:
<template v-for="k,i in state.Store.KeyNames">{{i?", ":""}}<code @click="forms.delKey.name=k">{{k}}</code></template>.</p>
</form>
</template>
</div>
</body>
</html>

35
html/ui/js/Cluster.js Normal file
View File

@ -0,0 +1,35 @@
import Downloads from './Downloads.js';
import GetCopy from './GetCopy.js';
export default {
components: { Downloads, GetCopy },
props: [ 'cluster', 'token', 'state' ],
template: `
<div class="cluster">
<div class="title">Cluster {{ cluster.Name }}</div>
<div class="section">Tokens</div>
<section class="links">
<GetCopy v-for="n in cluster.Tokens" :token="token" :name="n" :href="'/clusters/'+cluster.Name+'/tokens/'+n" />
</section>
<div class="section">Passwords</div>
<section class="links">
<GetCopy v-for="n in cluster.Passwords" :token="token" :name="n" :href="'/clusters/'+cluster.Name+'/passwords/'+n" />
</section>
<div class="section">Downloads</div>
<section class="downloads">
<Downloads :token="token" :state="state" kind="cluster" :name="cluster.Name" />
</section>
<div class="section">CAs</div>
<table><tr><th>Name</th><th>Certificate</th><th>Signed certificates</th></tr>
<tr v-for="ca in cluster.CAs">
<td>{{ ca.Name }}</td>
<td><GetCopy :token="token" name="cert" :href="'/clusters/'+cluster.Name+'/CAs/'+ca.Name+'/certificate'" /></td>
<td><template v-for="signed in ca.Signed">
{{" "}}
<GetCopy :token="token" :name="signed" :href="'/clusters/'+cluster.Name+'/CAs/'+ca.Name+'/signed?name='+signed" />
</template></td>
</tr></table>
</div>
`
}

64
html/ui/js/Downloads.js Normal file
View File

@ -0,0 +1,64 @@
export default {
props: [ 'kind', 'name', 'token', 'state' ],
data() {
return { createDisabled: false, selectedAssets: {} }
},
computed: {
availableAssets() {
return {
cluster: ['addons'],
host: [
"kernel",
"initrd",
"bootstrap.tar",
"boot.img.lz4",
"boot.iso",
"config",
"bootstrap-config",
"boot.tar",
"boot-efi.tar",
"boot.img.gz",
"boot.img",
"ipxe",
],
}[this.kind]
},
downloads() {
let ret = []
Object.entries(this.state.Downloads)
.filter(e => { let d=e[1]; return d.Kind == this.kind && d.Name == this.name })
.forEach(e => {
let token= e[0], d = e[1]
d.Assets.forEach(asset => {
ret.push({name: asset, url: '/public/downloads/'+token+'/'+asset})
})
})
return ret
},
assets() {
return this.availableAssets.filter(a => this.selectedAssets[a])
},
},
methods: {
createToken() {
event.preventDefault()
this.createDisabled = true
fetch('/authorize-download', {
method: 'POST',
body: JSON.stringify({Kind: this.kind, Name: this.name, Assets: this.assets}),
headers: { 'Authorization': 'Bearer ' + this.token, 'Content-Type': 'application/json' },
}).then((resp) => resp.json())
.then((token) => { this.selectedAssets = {}; this.createDisabled = false })
.catch((e) => { alert('failed to create link'); this.createDisabled = false })
},
},
template: `<div class="downloads">
<div class="options">
<span v-for="asset in availableAssets"><label><input type="checkbox" v-model="selectedAssets[asset]" />&nbsp;{{ asset }}</label></span>
</div>
<button :disabled="createDisabled || assets.length==0" @click="createToken">+</button>
<div><a v-for="d in downloads" target="_blank" :href="d.url">{{ d.name }}</a></div>
</div>`
}

32
html/ui/js/GetCopy.js Normal file
View File

@ -0,0 +1,32 @@
export default {
props: [ 'name', 'href', 'token' ],
data() { return {showCopied: false} },
template: `<span class="notif"><div v-if="showCopied">copied!</div><a :href="href" @click="fetchAndSave()">{{name}}</a>&nbsp;<a href="#" class="copy" @click="fetchAndCopy()">&#x1F5D0;</a></span>`,
methods: {
fetch() {
event.preventDefault()
return fetch(this.href, {
method: 'GET',
headers: { 'Authorization': 'Bearer ' + this.token },
})
},
handleFetchError(e) {
console.log("failed to get value:", e)
alert('failed to get value')
},
fetchAndSave() {
this.fetch().then(resp => resp.blob()).then((value) => {
window.open(URL.createObjectURL(value), "_blank")
}).catch(this.handleFetchError)
},
fetchAndCopy() {
this.fetch()
.then((resp) => resp.headers.get("content-type") == "application/json" ? resp.json() : resp.text())
.then((value) => {
window.navigator.clipboard.writeText(value)
this.showCopied = true
setTimeout(() => { this.showCopied = false }, 1000)
}).catch(this.handleFetchError)
},
},
}

21
html/ui/js/Host.js Normal file
View File

@ -0,0 +1,21 @@
import Downloads from './Downloads.js';
export default {
components: { Downloads },
props: [ 'host', 'token', 'state' ],
template: `
<div class="host">
<div class="title">Host {{ host.Name }}</div>
<section>
<template v-for="ip in host.IPs">
{{ ip }}
</template>
</section>
<div class="section">Downloads</div>
<section>
<Downloads :token="token" :state="state" kind="host" :name="host.Name" />
</section>
</div>
`
}

199
html/ui/js/app.js Normal file
View File

@ -0,0 +1,199 @@
import { createApp } from './vue.esm-browser.js';
import Cluster from './Cluster.js';
import Host from './Host.js';
createApp({
components: { Cluster, Host },
data() {
return {
forms: {
store: {},
storeUpload: {},
delKey: {},
},
session: {},
error: null,
publicState: null,
serverVersion: null,
uiHash: null,
watchingState: false,
state: null,
}
},
mounted() {
this.session = JSON.parse(sessionStorage.state || "{}")
this.watchPublicState()
},
watch: {
session: {
deep: true,
handler(v) {
sessionStorage.state = JSON.stringify(v)
if (v.token && !this.watchingState) {
this.watchState()
this.watchingState = true
}
}
},
publicState: {
deep: true,
handler(v) {
if (v) {
this.serverVersion = v.ServerVersion
if (this.uiHash && v.UIHash != this.uiHash) {
console.log("reloading")
location.reload()
} else {
this.uiHash = v.UIHash
}
}
},
}
},
methods: {
copyText(text) {
event.preventDefault()
window.navigator.clipboard.writeText(text)
},
setToken() {
event.preventDefault()
this.session.token = this.forms.setToken
this.forms.setToken = null
},
uploadStore() {
event.preventDefault()
this.apiPost('/public/store.tar', this.$refs.storeUpload.files[0], (v) => {
this.forms.store = {}
}, "application/tar")
},
namedPassphrase(name, passphrase) {
return {Name: this.forms.store.name, Passphrase: btoa(this.forms.store.pass1)}
},
storeAddKey() {
this.apiPost('/store/add-key', this.namedPassphrase(), (v) => {
this.forms.store = {}
})
},
storeDelKey() {
let name = this.forms.delKey.name
if (!confirm("Remove key named "+JSON.stringify(name)+"?")) {
return
}
this.apiPost('/store/delete-key', name , (v) => {
this.forms.delKey = {}
})
},
unlockStore() {
this.apiPost('/public/unlock-store', this.namedPassphrase(), (v) => {
this.forms.store = {}
if (v) {
this.session.token = v
if (!this.watchingState) {
this.watchState()
this.watchingState = true
}
}
})
},
uploadConfig() {
event.preventDefault()
this.apiPost('/configs', this.$refs.configUpload.files[0], (v) => {}, "text/vnd.yaml")
},
apiPost(action, data, onload, contentType = 'application/json') {
event.preventDefault()
if (data === undefined) {
throw("action " + action + ": no data")
}
/* TODO
fetch(action, {
method: 'POST',
body: JSON.stringify(data),
})
.then((response) => response.json())
.then((result) => onload)
// */
var xhr = new XMLHttpRequest()
xhr.responseType = 'json'
// TODO spinner, pending action notification, or something
xhr.onerror = () => {
// this.actionResults.splice(idx, 1, {...item, done: true, failed: true })
}
xhr.onload = (r) => {
if (xhr.status != 200) {
this.error = xhr.response
return
}
// this.actionResults.splice(idx, 1, {...item, done: true, resp: xhr.responseText})
this.error = null
if (onload) {
onload(xhr.response)
}
}
xhr.open("POST", action)
xhr.setRequestHeader('Accept', 'application/json')
xhr.setRequestHeader('Content-Type', contentType)
if (this.session.token) {
xhr.setRequestHeader('Authorization', 'Bearer '+this.session.token)
}
if (contentType == "application/json") {
xhr.send(JSON.stringify(data))
} else {
xhr.send(data)
}
},
download(url) {
event.target.target = '_blank'
event.target.href = this.downloadLink(url)
},
downloadLink(url) {
// TODO once-shot download link
return url + '?token=' + this.session.token
},
watchPublicState() {
this.watchStream('publicState', '/public-state')
},
watchState() {
this.watchStream('state', '/state', true)
},
watchStream(field, path, withToken) {
let evtSrc = new EventSource(path + (withToken ? '?token='+this.session.token : ''));
evtSrc.onmessage = (e) => {
let update = JSON.parse(e.data)
console.log("watch "+path+":", update)
if (update.err) {
console.log("watch error from server:", err)
}
if (update.set) {
this[field] = update.set
}
if (update.p) { // patch
new jsonpatch.JSONPatch(update.p, true).apply(this[field])
}
}
evtSrc.onerror = (e) => {
// console.log("event source " + path + " error:", e)
if (evtSrc) evtSrc.close()
this[field] = null
window.setTimeout(() => { this.watchStream(field, path, withToken) }, 1000)
}
},
}
}).mount('#app')

36
html/ui/js/jsonpatch.min.js vendored Normal file

File diff suppressed because one or more lines are too long

16172
html/ui/js/vue.esm-browser.js Normal file

File diff suppressed because it is too large Load Diff

152
html/ui/style.css Normal file
View File

@ -0,0 +1,152 @@
body {
background: white;
}
button[disabled] {
opacity: 0.5;
}
a[href], a[href]:visited, button.link {
border: none;
color: blue;
background: none;
cursor: pointer;
text-decoration: none;
}
table {
border-collapse: collapse;
}
th, td {
border-left: dotted 1pt;
border-right: dotted 1pt;
border-bottom: dotted 1pt;
padding: 2pt 4pt;
}
tr:first-child > th {
border-top: dotted 1pt;
}
th, tr:last-child > td {
border-bottom: solid 1pt;
}
.flat > * { margin-left: 1ex; }
.flat > *:first-child { margin-left: 0; }
.green { color: green; }
.red { color: red; }
@media (prefers-color-scheme: dark) {
body {
background: black;
color: orange;
}
button, input[type=submit] {
background: #333;
color: #eee;
}
a[href], a[href]:visited, button.link {
border: none;
color: #31b0fa;
}
.red { color: #c00; }
}
header {
display: flex;
align-items: center;
border-bottom: 2pt solid;
margin: 0 0 1em 0;
padding: 1ex;
justify-content: space-between;
}
#logo > img {
vertical-align: middle;
}
header .utils > * {
margin-left: 1ex;
}
.error {
display: flex;
position: relative;
background: rgba(255,0,0,0.2);
border: 1pt solid red;
justify-content: space-between;
align-items: center;
}
.error .btn-close,
.error .code {
background: #600;
color: white;
font-weight: bold;
border: none;
align-self: stretch;
padding: 1ex 1em;
}
.error .code {
order: 1;
display: flex;
align-items: center;
text-align: center;
}
.error .message {
order: 2;
padding: 1ex 2em;
}
.error .btn-close {
order: 3;
}
.sheets {
display: flex;
align-items: stretch;
}
.sheets > div {
margin: 0 1ex;
border: 1pt solid;
border-radius: 6pt;
}
.sheets .title {
text-align: center;
font-weight: bold;
font-size: large;
padding: 2pt 6pt;
background: rgba(127,127,127,0.5);
}
.sheets .section {
padding: 2pt 6pt 2pt 6pt;
font-weight: bold;
border-top: 1px dotted;
}
.sheets section {
margin: 2pt 6pt 6pt 6pt;
}
.sheets > *:last-child > table:last-child > tr:last-child > td {
border-bottom: none;
}
.notif {
display: inline-block;
position: relative;
}
.notif > div:first-child {
position: absolute;
min-width: 100%; height: 100%;
background: white;
opacity: 75%;
text-align: center;
}
.links > * { margin-left: 1ex; }
.links > *:first-child { margin-left: 0; }
@media (prefers-color-scheme: dark) {
.notif > div:first-child {
background: black;
}
}
.copy { font-size: small; }

2
install Executable file
View File

@ -0,0 +1,2 @@
#! /bin/sh
go install -trimpath ./cmd/...

View File

@ -1,6 +1,26 @@
**/*.go Dockerfile { modd.conf {}
**/*.go go.mod go.sum {
prep: go test ./... prep: go test ./...
prep: go install ./cmd/... prep: mkdir -p dist
prep: docker build -t dls . prep: hack/build ./...
#prep: docker build --build-arg GOPROXY=$GOPROXY -t dls .
#daemon +sigterm: /var/lib/direktil/test-run #daemon +sigterm: /var/lib/direktil/test-run
} }
html/**/* {
prep: hack/build ./cmd/dkl-local-server
}
dist/dkl-local-server {
prep: mkdir -p tmp
#daemon +sigterm: dist/dkl-local-server -data tmp -auto-unlock test
}
dist/dkl-dir2config {
prep: dist/dkl-dir2config --debug --in test-dir2config
}
**/*.proto !dist/**/* {
prep: for mod in @mods; do protoc -I ./ --go_out=plugins=grpc,paths=source_relative:. $mod; done
}

View File

@ -1,22 +1,33 @@
package clustersconfig package clustersconfig
import ( import (
"flag"
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
"log"
"net" "net"
"os"
"path/filepath"
"strings" "strings"
"text/template" "text/template"
yaml "gopkg.in/yaml.v2" yaml "gopkg.in/yaml.v2"
) )
var (
templateDetailsDir = flag.String("template-details-dir",
filepath.Join(os.TempDir(), "dkl-dir2config"),
"write details of template execute in this dir")
templateID = 0
)
type Config struct { type Config struct {
Hosts []*Host Hosts []*Host
Groups []*Group
Clusters []*Cluster Clusters []*Cluster
Configs []*Template Configs []*Template
StaticPods []*Template `yaml:"static_pods"` StaticPods map[string][]*Template `yaml:"static_pods"`
Addons map[string][]*Template Addons map[string][]*Template
SSLConfig string `yaml:"ssl_config"` SSLConfig string `yaml:"ssl_config"`
CertRequests []*CertRequest `yaml:"cert_requests"` CertRequests []*CertRequest `yaml:"cert_requests"`
@ -76,15 +87,6 @@ func (c *Config) HostByMAC(mac string) *Host {
return nil return nil
} }
func (c *Config) Group(name string) *Group {
for _, group := range c.Groups {
if group.Name == name {
return group
}
}
return nil
}
func (c *Config) Cluster(name string) *Cluster { func (c *Config) Cluster(name string) *Cluster {
for _, cluster := range c.Clusters { for _, cluster := range c.Clusters {
if cluster.Name == name { if cluster.Name == name {
@ -103,15 +105,6 @@ func (c *Config) ConfigTemplate(name string) *Template {
return nil return nil
} }
func (c *Config) StaticPodsTemplate(name string) *Template {
for _, s := range c.StaticPods {
if s.Name == name {
return s
}
}
return nil
}
func (c *Config) CSR(name string) *CertRequest { func (c *Config) CSR(name string) *CertRequest {
for _, s := range c.CertRequests { for _, s := range c.CertRequests {
if s.Name == name { if s.Name == name {
@ -133,59 +126,91 @@ func (c *Config) SaveTo(path string) error {
type Template struct { type Template struct {
Name string Name string
Template string Template string
parsedTemplate *template.Template
} }
func (t *Template) Execute(wr io.Writer, data interface{}, extraFuncs map[string]interface{}) error { func (t *Template) Execute(contextName, elementName string, wr io.Writer, data interface{}, extraFuncs map[string]interface{}) error {
if t.parsedTemplate == nil { var templateFuncs = map[string]interface{}{
var templateFuncs = map[string]interface{}{ "indent": func(indent, s string) (indented string) {
"indent": func(indent, s string) (indented string) { indented = indent + strings.Replace(s, "\n", "\n"+indent, -1)
indented = indent + strings.Replace(s, "\n", "\n"+indent, -1) return
return },
}, "yaml": func(v any) (s string, err error) {
ba, err := yaml.Marshal(v)
s = string(ba)
return
},
}
for name, f := range extraFuncs {
templateFuncs[name] = f
}
tmpl, err := template.New(t.Name).
Funcs(templateFuncs).
Parse(t.Template)
if err != nil {
return err
}
if *templateDetailsDir != "" {
templateID++
base := filepath.Join(*templateDetailsDir, contextName, fmt.Sprintf("%s-%03d", elementName, templateID))
os.MkdirAll(base, 0700)
base += string(filepath.Separator)
log.Print("writing template details: ", base, "{in,data,out}")
if err := ioutil.WriteFile(base+"in", []byte(t.Template), 0600); err != nil {
return err
} }
for name, f := range extraFuncs { yamlBytes, err := yaml.Marshal(data)
templateFuncs[name] = f
}
tmpl, err := template.New(t.Name).
Funcs(templateFuncs).
Parse(t.Template)
if err != nil { if err != nil {
return err return err
} }
t.parsedTemplate = tmpl
if err := ioutil.WriteFile(base+"data", yamlBytes, 0600); err != nil {
return err
}
out, err := os.Create(base + "out")
if err != nil {
return err
}
defer out.Close()
wr = io.MultiWriter(wr, out)
} }
return t.parsedTemplate.Execute(wr, data) return tmpl.Execute(wr, data)
} }
// Host represents a host served by this server. // Host represents a host served by this server.
type Host struct { type Host struct {
WithRev WithRev
Name string
Name string
Labels map[string]string
Annotations map[string]string
MAC string MAC string
IP string IP string
IPs []string IPs []string
Cluster string Cluster string
Group string Group string
Vars Vars
}
// Group represents a group of hosts and provides their configuration. IPXE string
type Group struct { Kernel string
WithRev Initrd string
Name string BootstrapConfig string `yaml:"bootstrap_config"`
Master bool Config string
IPXE string Versions map[string]string
Kernel string
Initrd string
Config string
StaticPods string `yaml:"static_pods"` StaticPods string `yaml:"static_pods"`
Versions map[string]string
Vars Vars Vars Vars
} }
// Vars store user-defined key-values // Vars store user-defined key-values
@ -194,13 +219,18 @@ type Vars map[string]interface{}
// Cluster represents a cluster of hosts, allowing for cluster-wide variables. // Cluster represents a cluster of hosts, allowing for cluster-wide variables.
type Cluster struct { type Cluster struct {
WithRev WithRev
Name string
Name string
Labels map[string]string
Annotations map[string]string
Domain string Domain string
Addons string Addons []string
Subnets struct { Subnets struct {
Services string Services string
Pods string Pods string
} }
Vars Vars Vars Vars
} }
@ -215,7 +245,7 @@ func (c *Cluster) DNSSvcIP() net.IP {
func (c *Cluster) NthSvcIP(n byte) net.IP { func (c *Cluster) NthSvcIP(n byte) net.IP {
_, cidr, err := net.ParseCIDR(c.Subnets.Services) _, cidr, err := net.ParseCIDR(c.Subnets.Services)
if err != nil { if err != nil {
panic(fmt.Errorf("Invalid services CIDR: %v", err)) panic(fmt.Errorf("invalid services CIDR: %v", err))
} }
ip := cidr.IP ip := cidr.IP

View File

@ -70,6 +70,7 @@ func (d *Defaults) Load(dir, suffix string, value Rev, data []byte) (err error)
} }
func (d *Defaults) Open(rev, filePath string) (rd io.Reader, err error) { func (d *Defaults) Open(rev, filePath string) (rd io.Reader, err error) {
log.Printf("openning defaults at %s:%s", rev, filePath)
tree, err := d.treeAt(rev) tree, err := d.treeAt(rev)
if err != nil { if err != nil {
return return
@ -94,12 +95,17 @@ func (d *Defaults) ReadAll(rev, filePath string) (ba []byte, err error) {
} }
func (d *Defaults) List(rev, dir string) (names []string, err error) { func (d *Defaults) List(rev, dir string) (names []string, err error) {
log.Printf("listing defaults at %s:%s", rev, dir)
tree, err := d.treeAt(rev) tree, err := d.treeAt(rev)
if err != nil { if err != nil {
return return
} }
dirPrefix := dir + "/"
err = tree.Files().ForEach(func(f *object.File) (err error) { err = tree.Files().ForEach(func(f *object.File) (err error) {
if !strings.HasPrefix(f.Name, dirPrefix) {
return
}
if !strings.HasSuffix(f.Name, ".yaml") { if !strings.HasSuffix(f.Name, ".yaml") {
return return
} }
@ -126,6 +132,11 @@ func (d *Defaults) treeAt(rev string) (tree *object.Tree, err error) {
obj, err = o.Object() obj, err = o.Object()
case *object.Commit: // commit -> tree case *object.Commit: // commit -> tree
msg := o.Message
if len(msg) > 30 {
msg = msg[:27] + "..."
}
log.Printf("open defaults at commit %s: %s", o.Hash.String()[:7], msg)
return o.Tree() return o.Tree()
default: default:

View File

@ -3,7 +3,6 @@ package clustersconfig
import ( import (
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"log"
"os" "os"
"path" "path"
"path/filepath" "path/filepath"
@ -15,37 +14,38 @@ import (
// Debug enables debug logs from this package. // Debug enables debug logs from this package.
var Debug = false var Debug = false
func FromDir(dirPath, defaultsPath string) (*Config, error) { func FromDir(
if Debug { read func(path string) ([]byte, error),
log.Printf("loading config from dir %s (defaults from %s)", dirPath, defaultsPath) assemble func(path string) ([]byte, error),
} listBase func(path string) ([]string, error),
listMerged func(path string) ([]string, error),
) (*Config, error) {
defaults, err := NewDefaults(defaultsPath) load := func(dir, name string, out any) (err error) {
if err != nil { ba, err := assemble(filepath.Join(dir, name))
return nil, fmt.Errorf("failed to load defaults: %v", err)
}
store := &dirStore{dirPath}
load := func(dir, name string, out Rev) error {
ba, err := store.Get(path.Join(dir, name))
if err != nil { if err != nil {
return fmt.Errorf("failed to load %s/%s from dir: %v", dir, name, err) return
} }
if err = defaults.Load(dir, ".yaml", out, ba); err != nil { err = yaml.UnmarshalStrict(ba, out)
return fmt.Errorf("failed to enrich %s/%s from defaults: %v", dir, name, err) if err != nil {
return
} }
return nil return nil
} }
config := &Config{Addons: make(map[string][]*Template)} config := &Config{
Addons: make(map[string][]*Template),
StaticPods: make(map[string][]*Template),
}
// load clusters // load clusters
names, err := store.List("clusters") names, err := listBase("clusters")
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to list clusters: %v", err) return nil, fmt.Errorf("failed to list clusters: %v", err)
} }
for _, name := range names { for _, name := range names {
name, _ = strings.CutSuffix(name, ".yaml")
cluster := &Cluster{Name: name} cluster := &Cluster{Name: name}
if err := load("clusters", name, cluster); err != nil { if err := load("clusters", name, cluster); err != nil {
return nil, err return nil, err
@ -54,97 +54,14 @@ func FromDir(dirPath, defaultsPath string) (*Config, error) {
config.Clusters = append(config.Clusters, cluster) config.Clusters = append(config.Clusters, cluster)
} }
// load groups
names, err = store.List("groups")
if err != nil {
return nil, fmt.Errorf("failed to list groups: %v", err)
}
read := func(rev, filePath string) (data []byte, fromDefaults bool, err error) {
data, err = store.Get(filePath)
if err != nil {
err = fmt.Errorf("faild to read %s: %v", filePath, err)
return
}
if data != nil {
return // ok
}
if len(rev) == 0 {
err = fmt.Errorf("entry not found: %s", filePath)
return
}
data, err = defaults.ReadAll(rev, filePath+".yaml")
if err != nil {
err = fmt.Errorf("failed to read %s:%s: %v", rev, filePath, err)
return
}
fromDefaults = true
return
}
template := func(rev, dir, name string, templates *[]*Template) (ref string, err error) {
ref = name
if len(name) == 0 {
return
}
ba, fromDefaults, err := read(rev, path.Join(dir, name))
if err != nil {
return
}
if fromDefaults {
ref = rev + ":" + name
}
if !hasTemplate(ref, *templates) {
if Debug {
log.Printf("new template in %s: %s", dir, ref)
}
*templates = append(*templates, &Template{
Name: ref,
Template: string(ba),
})
}
return
}
for _, name := range names {
group := &Group{Name: name}
if err := load("groups", name, group); err != nil {
return nil, err
}
group.Config, err = template(group.Rev(), "configs", group.Config, &config.Configs)
if err != nil {
return nil, fmt.Errorf("failed to load config for group %q: %v", name, err)
}
if Debug {
log.Printf("group %q: config=%q static_pods=%q", group.Name, group.Config, group.StaticPods)
}
group.StaticPods, err = template(group.Rev(), "static-pods", group.StaticPods, &config.StaticPods)
if err != nil {
return nil, fmt.Errorf("failed to load static pods for group %q: %v", name, err)
}
config.Groups = append(config.Groups, group)
}
// load hosts // load hosts
names, err = store.List("hosts") names, err = listBase("hosts")
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to list hosts: %v", err) return nil, fmt.Errorf("failed to list hosts: %v", err)
} }
for _, name := range names { for _, name := range names {
name, _ = strings.CutSuffix(name, ".yaml")
o := &Host{Name: name} o := &Host{Name: name}
if err := load("hosts", name, o); err != nil { if err := load("hosts", name, o); err != nil {
return nil, err return nil, err
@ -154,28 +71,20 @@ func FromDir(dirPath, defaultsPath string) (*Config, error) {
} }
// load config templates // load config templates
loadTemplates := func(rev, dir string, templates *[]*Template) error { loadTemplates := func(dir string, templates *[]*Template) error {
names, err := store.List(dir) names, err := listMerged(dir)
if err != nil { if err != nil {
return fmt.Errorf("failed to list %s: %v", dir, err) return fmt.Errorf("failed to list %s: %v", dir, err)
} }
if len(rev) != 0 { for _, fullName := range names {
var defaultsNames []string name, _ := strings.CutSuffix(fullName, ".yaml")
defaultsNames, err = defaults.List(rev, dir)
if err != nil {
return fmt.Errorf("failed to list %s:%s: %v", rev, dir, err)
}
names = append(names, defaultsNames...)
}
for _, name := range names {
if hasTemplate(name, *templates) { if hasTemplate(name, *templates) {
continue continue
} }
ba, _, err := read(rev, path.Join(dir, name)) ba, err := read(path.Join(dir, fullName))
if err != nil { if err != nil {
return err return err
} }
@ -189,33 +98,57 @@ func FromDir(dirPath, defaultsPath string) (*Config, error) {
return nil return nil
} }
loadTemplates("configs", &config.Configs)
// cluster addons
for _, cluster := range config.Clusters { for _, cluster := range config.Clusters {
addonSet := cluster.Addons addonSets := cluster.Addons
if len(addonSet) == 0 { if len(addonSets) == 0 {
continue continue
} }
if _, ok := config.Addons[addonSet]; ok { for _, addonSet := range addonSets {
if _, ok := config.Addons[addonSet]; ok {
continue
}
templates := make([]*Template, 0)
if err = loadTemplates(path.Join("addons", addonSet), &templates); err != nil {
return nil, err
}
config.Addons[addonSet] = templates
}
}
// cluster static pods
for _, host := range config.Hosts {
bpSet := host.StaticPods
if bpSet == "" {
continue
}
if _, ok := config.StaticPods[bpSet]; ok {
continue continue
} }
templates := make([]*Template, 0) templates := make([]*Template, 0)
if err = loadTemplates(cluster.Rev(), path.Join("addons", addonSet), &templates); err != nil { if err = loadTemplates(path.Join("static-pods", bpSet), &templates); err != nil {
return nil, err return nil, err
} }
config.Addons[addonSet] = templates config.StaticPods[bpSet] = templates
} }
// load SSL configuration // load SSL configuration
if ba, err := ioutil.ReadFile(filepath.Join(dirPath, "ssl-config.json")); err == nil { if ba, err := read("ssl-config.json"); err == nil {
config.SSLConfig = string(ba) config.SSLConfig = string(ba)
} else if !os.IsNotExist(err) { } else if !os.IsNotExist(err) {
return nil, err return nil, err
} }
if ba, err := ioutil.ReadFile(filepath.Join(dirPath, "cert-requests.yaml")); err == nil { if ba, err := read("cert-requests.yaml"); err == nil {
reqs := make([]*CertRequest, 0) reqs := make([]*CertRequest, 0)
if err = yaml.Unmarshal(ba, &reqs); err != nil { if err = yaml.Unmarshal(ba, &reqs); err != nil {
return nil, err return nil, err

View File

@ -0,0 +1,61 @@
package config
type Config struct {
AntiPhishingCode string `json:"anti_phishing_code"`
Keymap string
Modules string
Auths []Auth
Networks []struct {
Name string
Interfaces []struct {
Var string
N int
Regexps []string
}
Script string
}
LVM []LvmVG
Bootstrap Bootstrap
}
type Auth struct {
Name string
SSHKey string `yaml:"sshKey"`
Password string `yaml:"password"`
}
type LvmVG struct {
VG string
PVs struct {
N int
Regexps []string
}
Defaults struct {
FS string
Raid *RaidConfig
}
LVs []struct {
Name string
Crypt string
FS string
Raid *RaidConfig
Size string
Extents string
}
}
type RaidConfig struct {
Mirrors int
Stripes int
}
type Bootstrap struct {
Dev string
Seed string
}

View File

@ -1,10 +1,13 @@
package mime package mime
const ( const (
YAML = "text/vnd.yaml" JSON = "application/json"
TAR = "application/tar" YAML = "text/vnd.yaml"
DISK = "application/x-diskimage" TAR = "application/tar"
ISO = "application/x-iso9660-image" DISK = "application/x-diskimage"
IPXE = "text/x-ipxe" ISO = "application/x-iso9660-image"
OCTET = "application/octet-stream" IPXE = "text/x-ipxe"
OCTET = "application/octet-stream"
CERT = "application/x-x509-user-cert"
CACERT = "application/x-x509-ca-cert"
) )

29
pkg/utf16/utf16.go Normal file
View File

@ -0,0 +1,29 @@
package utf16
import (
"encoding/binary"
"fmt"
"unicode/utf8"
)
func FromUTF8(data []byte) (res []byte) {
endian := binary.LittleEndian
res = make([]byte, (len(data)+1)*2)
res = res[:2]
endian.PutUint16(res, 0xfeff)
for len(data) > 0 {
r, size := utf8.DecodeRune(data)
if r > 65535 {
panic(fmt.Errorf("r=0x%x > 0xffff", r))
}
slen := len(res)
res = res[:slen+2]
endian.PutUint16(res[slen:], uint16(r))
data = data[size:]
}
return
}

30
secretstore/io.go Normal file
View File

@ -0,0 +1,30 @@
package secretstore
import (
"crypto/rand"
"encoding/binary"
"fmt"
"io"
)
func readFull(in io.Reader, ba []byte) (err error) {
_, err = io.ReadFull(in, ba)
return
}
func read[T any](in io.Reader) (v T, err error) {
err = binary.Read(in, binary.BigEndian, &v)
return
}
var readSize = read[uint16]
func randRead(ba []byte) (err error) {
err = readFull(rand.Reader, ba)
if err != nil {
err = fmt.Errorf("failed to read random bytes: %w", err)
return
}
return
}

9
secretstore/mem.go Normal file
View File

@ -0,0 +1,9 @@
package secretstore
func Memzero(ba []byte) { memzero(ba) }
func memzero(ba []byte) {
for i := range ba {
ba[i] = 0
}
}

68
secretstore/reader.go Normal file
View File

@ -0,0 +1,68 @@
package secretstore
import (
"crypto/aes"
"crypto/cipher"
"io"
)
func (s *Store) NewReader(reader io.Reader) (r io.Reader, err error) {
iv := [aes.BlockSize]byte{}
err = readFull(reader, iv[:])
if err != nil {
return
}
r = storeReader{reader, s.NewDecrypter(iv)}
return
}
type storeReader struct {
reader io.Reader
decrypter cipher.Stream
}
func (r storeReader) Read(ba []byte) (n int, err error) {
n, err = r.reader.Read(ba)
if n > 0 {
r.decrypter.XORKeyStream(ba[:n], ba[:n])
}
return
}
func (s *Store) NewWriter(writer io.Writer) (r io.Writer, err error) {
iv := [aes.BlockSize]byte{}
if err = randRead(iv[:]); err != nil {
return
}
_, err = writer.Write(iv[:])
if err != nil {
return
}
r = storeWriter{writer, s.NewEncrypter(iv)}
return
}
type storeWriter struct {
writer io.Writer
encrypter cipher.Stream
}
func (r storeWriter) Write(ba []byte) (n int, err error) {
if len(ba) == 0 {
return
}
encBA := make([]byte, len(ba))
r.encrypter.XORKeyStream(encBA, ba)
n, err = r.writer.Write(encBA)
return
}

278
secretstore/secret-store.go Normal file
View File

@ -0,0 +1,278 @@
package secretstore
import (
"bufio"
"bytes"
"crypto/aes"
"crypto/cipher"
"crypto/sha512"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"os"
"strconv"
"syscall"
"golang.org/x/crypto/argon2"
)
type Store struct {
Salt [aes.BlockSize]byte
Keys []KeyEntry
unlocked bool
key [32]byte
}
type KeyEntry struct {
Name string
Hash [64]byte
EncKey [32]byte
}
func New() (s *Store) {
s = &Store{}
syscall.Mlock(s.key[:])
return
}
func Open(path string) (s *Store, err error) {
f, err := os.Open(path)
if err != nil {
return
}
defer f.Close()
s = New()
_, err = s.ReadFrom(bufio.NewReader(f))
return
}
func (s *Store) SaveTo(path string) (err error) {
f, err := os.OpenFile(path, syscall.O_CREAT|syscall.O_TRUNC|syscall.O_WRONLY, 0600)
if err != nil {
return
}
defer f.Close()
out := bufio.NewWriter(f)
_, err = s.WriteTo(out)
if err != nil {
return
}
err = out.Flush()
if err != nil {
return
}
return
}
func (s *Store) Close() {
memzero(s.key[:])
syscall.Munlock(s.key[:])
s.unlocked = false
}
func (s *Store) IsNew() bool {
return len(s.Keys) == 0
}
func (s *Store) Unlocked() bool {
return s.unlocked
}
func (s *Store) Init(name string, passphrase []byte) (err error) {
err = randRead(s.key[:])
if err != nil {
return
}
err = randRead(s.Salt[:])
if err != nil {
return
}
s.AddKey(name, passphrase)
s.unlocked = true
return
}
var jsonFormatHdr = []byte("{json}")
func (s *Store) ReadFrom(in io.Reader) (n int64, err error) {
memzero(s.key[:])
s.unlocked = false
defer func() {
if err != nil {
log.Output(2, fmt.Sprintf("failed after %d bytes", n))
}
}()
readFull := func(ba []byte) {
var nr int
nr, err = io.ReadFull(in, ba)
n += int64(nr)
}
// read the file's start (json header or start of salt)
readFull(s.Salt[:len(jsonFormatHdr)])
if err != nil {
return
}
if !bytes.Equal(s.Salt[:len(jsonFormatHdr)], jsonFormatHdr) {
// old key file
// finish reading the salt
readFull(s.Salt[len(jsonFormatHdr):])
if err != nil {
return
}
// read the (encrypted) keys
s.Keys = make([]KeyEntry, 0)
for {
k := KeyEntry{Name: "key-" + strconv.Itoa(len(s.Keys))}
readFull(k.Hash[:])
if err != nil {
if err == io.EOF {
err = nil
}
return
}
readFull(k.EncKey[:])
if err != nil {
return
}
s.Keys = append(s.Keys, k)
}
}
err = json.NewDecoder(in).Decode(s)
return
}
func (s *Store) WriteTo(out io.Writer) (n int64, err error) {
_, err = out.Write(jsonFormatHdr)
if err != nil {
return
}
err = json.NewEncoder(out).Encode(s)
return
}
var ErrNoSuchKey = errors.New("no such key")
func (s *Store) HasKey(passphrase []byte) bool {
key, hash := s.keyPairFromPassword(passphrase)
defer memzero(key[:])
for _, k := range s.Keys {
if k.Hash == hash {
return true
}
}
return false
}
func (s *Store) Unlock(passphrase []byte) (ok bool) {
key, hash := s.keyPairFromPassword(passphrase)
defer memzero(key[:])
var idx = -1
for i := range s.Keys {
if hash == s.Keys[i].Hash {
idx = i
break
}
}
if idx == -1 {
return
}
s.decryptTo(s.key[:], s.Keys[idx].EncKey[:], &key)
s.unlocked = true
return true
}
func (s *Store) AddKey(name string, passphrase []byte) {
key, hash := s.keyPairFromPassword(passphrase)
memzero(passphrase)
defer memzero(key[:])
k := KeyEntry{Name: name, Hash: hash}
encKey := s.encrypt(s.key[:], &key)
copy(k.EncKey[:], encKey)
s.Keys = append(s.Keys, k)
}
func (s *Store) keyPairFromPassword(password []byte) (key [32]byte, hash [64]byte) {
keySlice := argon2.IDKey(password, s.Salt[:], 1, 64*1024, 4, 32)
copy(key[:], keySlice)
memzero(keySlice)
hash = sha512.Sum512(key[:])
return
}
func (s *Store) NewEncrypter(iv [aes.BlockSize]byte) cipher.Stream {
if !s.unlocked {
panic("not unlocked")
}
return newEncrypter(iv, &s.key)
}
func (s *Store) NewDecrypter(iv [aes.BlockSize]byte) cipher.Stream {
if !s.unlocked {
panic("not unlocked")
}
return newDecrypter(iv, &s.key)
}
func (s *Store) encrypt(src []byte, key *[32]byte) (dst []byte) {
dst = make([]byte, len(src))
newEncrypter(s.Salt, key).XORKeyStream(dst, src)
return
}
func (s *Store) decryptTo(dst []byte, src []byte, key *[32]byte) {
newDecrypter(s.Salt, key).XORKeyStream(dst, src)
}
func newEncrypter(iv [aes.BlockSize]byte, key *[32]byte) cipher.Stream {
c, err := aes.NewCipher(key[:])
if err != nil {
panic(fmt.Errorf("failed to init AES: %w", err))
}
return cipher.NewCFBEncrypter(c, iv[:])
}
func newDecrypter(iv [aes.BlockSize]byte, key *[32]byte) cipher.Stream {
c, err := aes.NewCipher(key[:])
if err != nil {
panic(fmt.Errorf("failed to init AES: %w", err))
}
return cipher.NewCFBDecrypter(c, iv[:])
}

19
update-boot.sh Executable file
View File

@ -0,0 +1,19 @@
#! /bin/bash
dls_url="$1"
set -ex
mount -o remount,rw /boot
if [ -e /boot/previous ]; then
rm -fr /boot/previous
fi
if [ -e /boot/current ]; then
mv /boot/current /boot/previous
fi
curl $dls_url/me/boot.tar |tar xv -C /boot
sync

View File

@ -1,5 +0,0 @@
*.sublime-*
.DS_Store
*.swp
*.swo
tags

View File

@ -1,7 +0,0 @@
language: go
go:
- 1.4
- 1.5
- 1.6
- tip

View File

@ -1,12 +0,0 @@
Copyright (c) 2012, Martin Angers
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View File

@ -1,187 +0,0 @@
# Purell
Purell is a tiny Go library to normalize URLs. It returns a pure URL. Pure-ell. Sanitizer and all. Yeah, I know...
Based on the [wikipedia paper][wiki] and the [RFC 3986 document][rfc].
[![build status](https://secure.travis-ci.org/PuerkitoBio/purell.png)](http://travis-ci.org/PuerkitoBio/purell)
## Install
`go get github.com/PuerkitoBio/purell`
## Changelog
* **2016-11-14 (v1.1.0)** : IDN: Conform to RFC 5895: Fold character width (thanks to @beeker1121).
* **2016-07-27 (v1.0.0)** : Normalize IDN to ASCII (thanks to @zenovich).
* **2015-02-08** : Add fix for relative paths issue ([PR #5][pr5]) and add fix for unnecessary encoding of reserved characters ([see issue #7][iss7]).
* **v0.2.0** : Add benchmarks, Attempt IDN support.
* **v0.1.0** : Initial release.
## Examples
From `example_test.go` (note that in your code, you would import "github.com/PuerkitoBio/purell", and would prefix references to its methods and constants with "purell."):
```go
package purell
import (
"fmt"
"net/url"
)
func ExampleNormalizeURLString() {
if normalized, err := NormalizeURLString("hTTp://someWEBsite.com:80/Amazing%3f/url/",
FlagLowercaseScheme|FlagLowercaseHost|FlagUppercaseEscapes); err != nil {
panic(err)
} else {
fmt.Print(normalized)
}
// Output: http://somewebsite.com:80/Amazing%3F/url/
}
func ExampleMustNormalizeURLString() {
normalized := MustNormalizeURLString("hTTpS://someWEBsite.com:443/Amazing%fa/url/",
FlagsUnsafeGreedy)
fmt.Print(normalized)
// Output: http://somewebsite.com/Amazing%FA/url
}
func ExampleNormalizeURL() {
if u, err := url.Parse("Http://SomeUrl.com:8080/a/b/.././c///g?c=3&a=1&b=9&c=0#target"); err != nil {
panic(err)
} else {
normalized := NormalizeURL(u, FlagsUsuallySafeGreedy|FlagRemoveDuplicateSlashes|FlagRemoveFragment)
fmt.Print(normalized)
}
// Output: http://someurl.com:8080/a/c/g?c=3&a=1&b=9&c=0
}
```
## API
As seen in the examples above, purell offers three methods, `NormalizeURLString(string, NormalizationFlags) (string, error)`, `MustNormalizeURLString(string, NormalizationFlags) (string)` and `NormalizeURL(*url.URL, NormalizationFlags) (string)`. They all normalize the provided URL based on the specified flags. Here are the available flags:
```go
const (
// Safe normalizations
FlagLowercaseScheme NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1
FlagLowercaseHost // http://HOST -> http://host
FlagUppercaseEscapes // http://host/t%ef -> http://host/t%EF
FlagDecodeUnnecessaryEscapes // http://host/t%41 -> http://host/tA
FlagEncodeNecessaryEscapes // http://host/!"#$ -> http://host/%21%22#$
FlagRemoveDefaultPort // http://host:80 -> http://host
FlagRemoveEmptyQuerySeparator // http://host/path? -> http://host/path
// Usually safe normalizations
FlagRemoveTrailingSlash // http://host/path/ -> http://host/path
FlagAddTrailingSlash // http://host/path -> http://host/path/ (should choose only one of these add/remove trailing slash flags)
FlagRemoveDotSegments // http://host/path/./a/b/../c -> http://host/path/a/c
// Unsafe normalizations
FlagRemoveDirectoryIndex // http://host/path/index.html -> http://host/path/
FlagRemoveFragment // http://host/path#fragment -> http://host/path
FlagForceHTTP // https://host -> http://host
FlagRemoveDuplicateSlashes // http://host/path//a///b -> http://host/path/a/b
FlagRemoveWWW // http://www.host/ -> http://host/
FlagAddWWW // http://host/ -> http://www.host/ (should choose only one of these add/remove WWW flags)
FlagSortQuery // http://host/path?c=3&b=2&a=1&b=1 -> http://host/path?a=1&b=1&b=2&c=3
// Normalizations not in the wikipedia article, required to cover tests cases
// submitted by jehiah
FlagDecodeDWORDHost // http://1113982867 -> http://66.102.7.147
FlagDecodeOctalHost // http://0102.0146.07.0223 -> http://66.102.7.147
FlagDecodeHexHost // http://0x42660793 -> http://66.102.7.147
FlagRemoveUnnecessaryHostDots // http://.host../path -> http://host/path
FlagRemoveEmptyPortSeparator // http://host:/path -> http://host/path
// Convenience set of safe normalizations
FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagEncodeNecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator
// For convenience sets, "greedy" uses the "remove trailing slash" and "remove www. prefix" flags,
// while "non-greedy" uses the "add (or keep) the trailing slash" and "add www. prefix".
// Convenience set of usually safe normalizations (includes FlagsSafe)
FlagsUsuallySafeGreedy NormalizationFlags = FlagsSafe | FlagRemoveTrailingSlash | FlagRemoveDotSegments
FlagsUsuallySafeNonGreedy NormalizationFlags = FlagsSafe | FlagAddTrailingSlash | FlagRemoveDotSegments
// Convenience set of unsafe normalizations (includes FlagsUsuallySafe)
FlagsUnsafeGreedy NormalizationFlags = FlagsUsuallySafeGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagRemoveWWW | FlagSortQuery
FlagsUnsafeNonGreedy NormalizationFlags = FlagsUsuallySafeNonGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagAddWWW | FlagSortQuery
// Convenience set of all available flags
FlagsAllGreedy = FlagsUnsafeGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
FlagsAllNonGreedy = FlagsUnsafeNonGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
)
```
For convenience, the set of flags `FlagsSafe`, `FlagsUsuallySafe[Greedy|NonGreedy]`, `FlagsUnsafe[Greedy|NonGreedy]` and `FlagsAll[Greedy|NonGreedy]` are provided for the similarly grouped normalizations on [wikipedia's URL normalization page][wiki]. You can add (using the bitwise OR `|` operator) or remove (using the bitwise AND NOT `&^` operator) individual flags from the sets if required, to build your own custom set.
The [full godoc reference is available on gopkgdoc][godoc].
Some things to note:
* `FlagDecodeUnnecessaryEscapes`, `FlagEncodeNecessaryEscapes`, `FlagUppercaseEscapes` and `FlagRemoveEmptyQuerySeparator` are always implicitly set, because internally, the URL string is parsed as an URL object, which automatically decodes unnecessary escapes, uppercases and encodes necessary ones, and removes empty query separators (an unnecessary `?` at the end of the url). So this operation cannot **not** be done. For this reason, `FlagRemoveEmptyQuerySeparator` (as well as the other three) has been included in the `FlagsSafe` convenience set, instead of `FlagsUnsafe`, where Wikipedia puts it.
* The `FlagDecodeUnnecessaryEscapes` decodes the following escapes (*from -> to*):
- %24 -> $
- %26 -> &
- %2B-%3B -> +,-./0123456789:;
- %3D -> =
- %40-%5A -> @ABCDEFGHIJKLMNOPQRSTUVWXYZ
- %5F -> _
- %61-%7A -> abcdefghijklmnopqrstuvwxyz
- %7E -> ~
* When the `NormalizeURL` function is used (passing an URL object), this source URL object is modified (that is, after the call, the URL object will be modified to reflect the normalization).
* The *replace IP with domain name* normalization (`http://208.77.188.166/ → http://www.example.com/`) is obviously not possible for a library without making some network requests. This is not implemented in purell.
* The *remove unused query string parameters* and *remove default query parameters* are also not implemented, since this is a very case-specific normalization, and it is quite trivial to do with an URL object.
### Safe vs Usually Safe vs Unsafe
Purell allows you to control the level of risk you take while normalizing an URL. You can aggressively normalize, play it totally safe, or anything in between.
Consider the following URL:
`HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid`
Normalizing with the `FlagsSafe` gives:
`https://www.root.com/toto/tE%1F///a/./b/../c/?z=3&w=2&a=4&w=1#invalid`
With the `FlagsUsuallySafeGreedy`:
`https://www.root.com/toto/tE%1F///a/c?z=3&w=2&a=4&w=1#invalid`
And with `FlagsUnsafeGreedy`:
`http://root.com/toto/tE%1F/a/c?a=4&w=1&w=2&z=3`
## TODOs
* Add a class/default instance to allow specifying custom directory index names? At the moment, removing directory index removes `(^|/)((?:default|index)\.\w{1,4})$`.
## Thanks / Contributions
@rogpeppe
@jehiah
@opennota
@pchristopher1275
@zenovich
@beeker1121
## License
The [BSD 3-Clause license][bsd].
[bsd]: http://opensource.org/licenses/BSD-3-Clause
[wiki]: http://en.wikipedia.org/wiki/URL_normalization
[rfc]: http://tools.ietf.org/html/rfc3986#section-6
[godoc]: http://go.pkgdoc.org/github.com/PuerkitoBio/purell
[pr5]: https://github.com/PuerkitoBio/purell/pull/5
[iss7]: https://github.com/PuerkitoBio/purell/issues/7

View File

@ -1,379 +0,0 @@
/*
Package purell offers URL normalization as described on the wikipedia page:
http://en.wikipedia.org/wiki/URL_normalization
*/
package purell
import (
"bytes"
"fmt"
"net/url"
"regexp"
"sort"
"strconv"
"strings"
"github.com/PuerkitoBio/urlesc"
"golang.org/x/net/idna"
"golang.org/x/text/unicode/norm"
"golang.org/x/text/width"
)
// A set of normalization flags determines how a URL will
// be normalized.
type NormalizationFlags uint
const (
// Safe normalizations
FlagLowercaseScheme NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1
FlagLowercaseHost // http://HOST -> http://host
FlagUppercaseEscapes // http://host/t%ef -> http://host/t%EF
FlagDecodeUnnecessaryEscapes // http://host/t%41 -> http://host/tA
FlagEncodeNecessaryEscapes // http://host/!"#$ -> http://host/%21%22#$
FlagRemoveDefaultPort // http://host:80 -> http://host
FlagRemoveEmptyQuerySeparator // http://host/path? -> http://host/path
// Usually safe normalizations
FlagRemoveTrailingSlash // http://host/path/ -> http://host/path
FlagAddTrailingSlash // http://host/path -> http://host/path/ (should choose only one of these add/remove trailing slash flags)
FlagRemoveDotSegments // http://host/path/./a/b/../c -> http://host/path/a/c
// Unsafe normalizations
FlagRemoveDirectoryIndex // http://host/path/index.html -> http://host/path/
FlagRemoveFragment // http://host/path#fragment -> http://host/path
FlagForceHTTP // https://host -> http://host
FlagRemoveDuplicateSlashes // http://host/path//a///b -> http://host/path/a/b
FlagRemoveWWW // http://www.host/ -> http://host/
FlagAddWWW // http://host/ -> http://www.host/ (should choose only one of these add/remove WWW flags)
FlagSortQuery // http://host/path?c=3&b=2&a=1&b=1 -> http://host/path?a=1&b=1&b=2&c=3
// Normalizations not in the wikipedia article, required to cover tests cases
// submitted by jehiah
FlagDecodeDWORDHost // http://1113982867 -> http://66.102.7.147
FlagDecodeOctalHost // http://0102.0146.07.0223 -> http://66.102.7.147
FlagDecodeHexHost // http://0x42660793 -> http://66.102.7.147
FlagRemoveUnnecessaryHostDots // http://.host../path -> http://host/path
FlagRemoveEmptyPortSeparator // http://host:/path -> http://host/path
// Convenience set of safe normalizations
FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagEncodeNecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator
// For convenience sets, "greedy" uses the "remove trailing slash" and "remove www. prefix" flags,
// while "non-greedy" uses the "add (or keep) the trailing slash" and "add www. prefix".
// Convenience set of usually safe normalizations (includes FlagsSafe)
FlagsUsuallySafeGreedy NormalizationFlags = FlagsSafe | FlagRemoveTrailingSlash | FlagRemoveDotSegments
FlagsUsuallySafeNonGreedy NormalizationFlags = FlagsSafe | FlagAddTrailingSlash | FlagRemoveDotSegments
// Convenience set of unsafe normalizations (includes FlagsUsuallySafe)
FlagsUnsafeGreedy NormalizationFlags = FlagsUsuallySafeGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagRemoveWWW | FlagSortQuery
FlagsUnsafeNonGreedy NormalizationFlags = FlagsUsuallySafeNonGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagAddWWW | FlagSortQuery
// Convenience set of all available flags
FlagsAllGreedy = FlagsUnsafeGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
FlagsAllNonGreedy = FlagsUnsafeNonGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
)
const (
defaultHttpPort = ":80"
defaultHttpsPort = ":443"
)
// Regular expressions used by the normalizations
var rxPort = regexp.MustCompile(`(:\d+)/?$`)
var rxDirIndex = regexp.MustCompile(`(^|/)((?:default|index)\.\w{1,4})$`)
var rxDupSlashes = regexp.MustCompile(`/{2,}`)
var rxDWORDHost = regexp.MustCompile(`^(\d+)((?:\.+)?(?:\:\d*)?)$`)
var rxOctalHost = regexp.MustCompile(`^(0\d*)\.(0\d*)\.(0\d*)\.(0\d*)((?:\.+)?(?:\:\d*)?)$`)
var rxHexHost = regexp.MustCompile(`^0x([0-9A-Fa-f]+)((?:\.+)?(?:\:\d*)?)$`)
var rxHostDots = regexp.MustCompile(`^(.+?)(:\d+)?$`)
var rxEmptyPort = regexp.MustCompile(`:+$`)
// Map of flags to implementation function.
// FlagDecodeUnnecessaryEscapes has no action, since it is done automatically
// by parsing the string as an URL. Same for FlagUppercaseEscapes and FlagRemoveEmptyQuerySeparator.
// Since maps have undefined traversing order, make a slice of ordered keys
var flagsOrder = []NormalizationFlags{
FlagLowercaseScheme,
FlagLowercaseHost,
FlagRemoveDefaultPort,
FlagRemoveDirectoryIndex,
FlagRemoveDotSegments,
FlagRemoveFragment,
FlagForceHTTP, // Must be after remove default port (because https=443/http=80)
FlagRemoveDuplicateSlashes,
FlagRemoveWWW,
FlagAddWWW,
FlagSortQuery,
FlagDecodeDWORDHost,
FlagDecodeOctalHost,
FlagDecodeHexHost,
FlagRemoveUnnecessaryHostDots,
FlagRemoveEmptyPortSeparator,
FlagRemoveTrailingSlash, // These two (add/remove trailing slash) must be last
FlagAddTrailingSlash,
}
// ... and then the map, where order is unimportant
var flags = map[NormalizationFlags]func(*url.URL){
FlagLowercaseScheme: lowercaseScheme,
FlagLowercaseHost: lowercaseHost,
FlagRemoveDefaultPort: removeDefaultPort,
FlagRemoveDirectoryIndex: removeDirectoryIndex,
FlagRemoveDotSegments: removeDotSegments,
FlagRemoveFragment: removeFragment,
FlagForceHTTP: forceHTTP,
FlagRemoveDuplicateSlashes: removeDuplicateSlashes,
FlagRemoveWWW: removeWWW,
FlagAddWWW: addWWW,
FlagSortQuery: sortQuery,
FlagDecodeDWORDHost: decodeDWORDHost,
FlagDecodeOctalHost: decodeOctalHost,
FlagDecodeHexHost: decodeHexHost,
FlagRemoveUnnecessaryHostDots: removeUnncessaryHostDots,
FlagRemoveEmptyPortSeparator: removeEmptyPortSeparator,
FlagRemoveTrailingSlash: removeTrailingSlash,
FlagAddTrailingSlash: addTrailingSlash,
}
// MustNormalizeURLString returns the normalized string, and panics if an error occurs.
// It takes an URL string as input, as well as the normalization flags.
func MustNormalizeURLString(u string, f NormalizationFlags) string {
result, e := NormalizeURLString(u, f)
if e != nil {
panic(e)
}
return result
}
// NormalizeURLString returns the normalized string, or an error if it can't be parsed into an URL object.
// It takes an URL string as input, as well as the normalization flags.
func NormalizeURLString(u string, f NormalizationFlags) (string, error) {
parsed, err := url.Parse(u)
if err != nil {
return "", err
}
if f&FlagLowercaseHost == FlagLowercaseHost {
parsed.Host = strings.ToLower(parsed.Host)
}
// The idna package doesn't fully conform to RFC 5895
// (https://tools.ietf.org/html/rfc5895), so we do it here.
// Taken from Go 1.8 cycle source, courtesy of bradfitz.
// TODO: Remove when (if?) idna package conforms to RFC 5895.
parsed.Host = width.Fold.String(parsed.Host)
parsed.Host = norm.NFC.String(parsed.Host)
if parsed.Host, err = idna.ToASCII(parsed.Host); err != nil {
return "", err
}
return NormalizeURL(parsed, f), nil
}
// NormalizeURL returns the normalized string.
// It takes a parsed URL object as input, as well as the normalization flags.
func NormalizeURL(u *url.URL, f NormalizationFlags) string {
for _, k := range flagsOrder {
if f&k == k {
flags[k](u)
}
}
return urlesc.Escape(u)
}
func lowercaseScheme(u *url.URL) {
if len(u.Scheme) > 0 {
u.Scheme = strings.ToLower(u.Scheme)
}
}
func lowercaseHost(u *url.URL) {
if len(u.Host) > 0 {
u.Host = strings.ToLower(u.Host)
}
}
func removeDefaultPort(u *url.URL) {
if len(u.Host) > 0 {
scheme := strings.ToLower(u.Scheme)
u.Host = rxPort.ReplaceAllStringFunc(u.Host, func(val string) string {
if (scheme == "http" && val == defaultHttpPort) || (scheme == "https" && val == defaultHttpsPort) {
return ""
}
return val
})
}
}
func removeTrailingSlash(u *url.URL) {
if l := len(u.Path); l > 0 {
if strings.HasSuffix(u.Path, "/") {
u.Path = u.Path[:l-1]
}
} else if l = len(u.Host); l > 0 {
if strings.HasSuffix(u.Host, "/") {
u.Host = u.Host[:l-1]
}
}
}
func addTrailingSlash(u *url.URL) {
if l := len(u.Path); l > 0 {
if !strings.HasSuffix(u.Path, "/") {
u.Path += "/"
}
} else if l = len(u.Host); l > 0 {
if !strings.HasSuffix(u.Host, "/") {
u.Host += "/"
}
}
}
func removeDotSegments(u *url.URL) {
if len(u.Path) > 0 {
var dotFree []string
var lastIsDot bool
sections := strings.Split(u.Path, "/")
for _, s := range sections {
if s == ".." {
if len(dotFree) > 0 {
dotFree = dotFree[:len(dotFree)-1]
}
} else if s != "." {
dotFree = append(dotFree, s)
}
lastIsDot = (s == "." || s == "..")
}
// Special case if host does not end with / and new path does not begin with /
u.Path = strings.Join(dotFree, "/")
if u.Host != "" && !strings.HasSuffix(u.Host, "/") && !strings.HasPrefix(u.Path, "/") {
u.Path = "/" + u.Path
}
// Special case if the last segment was a dot, make sure the path ends with a slash
if lastIsDot && !strings.HasSuffix(u.Path, "/") {
u.Path += "/"
}
}
}
func removeDirectoryIndex(u *url.URL) {
if len(u.Path) > 0 {
u.Path = rxDirIndex.ReplaceAllString(u.Path, "$1")
}
}
func removeFragment(u *url.URL) {
u.Fragment = ""
}
func forceHTTP(u *url.URL) {
if strings.ToLower(u.Scheme) == "https" {
u.Scheme = "http"
}
}
func removeDuplicateSlashes(u *url.URL) {
if len(u.Path) > 0 {
u.Path = rxDupSlashes.ReplaceAllString(u.Path, "/")
}
}
func removeWWW(u *url.URL) {
if len(u.Host) > 0 && strings.HasPrefix(strings.ToLower(u.Host), "www.") {
u.Host = u.Host[4:]
}
}
func addWWW(u *url.URL) {
if len(u.Host) > 0 && !strings.HasPrefix(strings.ToLower(u.Host), "www.") {
u.Host = "www." + u.Host
}
}
func sortQuery(u *url.URL) {
q := u.Query()
if len(q) > 0 {
arKeys := make([]string, len(q))
i := 0
for k, _ := range q {
arKeys[i] = k
i++
}
sort.Strings(arKeys)
buf := new(bytes.Buffer)
for _, k := range arKeys {
sort.Strings(q[k])
for _, v := range q[k] {
if buf.Len() > 0 {
buf.WriteRune('&')
}
buf.WriteString(fmt.Sprintf("%s=%s", k, urlesc.QueryEscape(v)))
}
}
// Rebuild the raw query string
u.RawQuery = buf.String()
}
}
func decodeDWORDHost(u *url.URL) {
if len(u.Host) > 0 {
if matches := rxDWORDHost.FindStringSubmatch(u.Host); len(matches) > 2 {
var parts [4]int64
dword, _ := strconv.ParseInt(matches[1], 10, 0)
for i, shift := range []uint{24, 16, 8, 0} {
parts[i] = dword >> shift & 0xFF
}
u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[2])
}
}
}
func decodeOctalHost(u *url.URL) {
if len(u.Host) > 0 {
if matches := rxOctalHost.FindStringSubmatch(u.Host); len(matches) > 5 {
var parts [4]int64
for i := 1; i <= 4; i++ {
parts[i-1], _ = strconv.ParseInt(matches[i], 8, 0)
}
u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[5])
}
}
}
func decodeHexHost(u *url.URL) {
if len(u.Host) > 0 {
if matches := rxHexHost.FindStringSubmatch(u.Host); len(matches) > 2 {
// Conversion is safe because of regex validation
parsed, _ := strconv.ParseInt(matches[1], 16, 0)
// Set host as DWORD (base 10) encoded host
u.Host = fmt.Sprintf("%d%s", parsed, matches[2])
// The rest is the same as decoding a DWORD host
decodeDWORDHost(u)
}
}
}
func removeUnncessaryHostDots(u *url.URL) {
if len(u.Host) > 0 {
if matches := rxHostDots.FindStringSubmatch(u.Host); len(matches) > 1 {
// Trim the leading and trailing dots
u.Host = strings.Trim(matches[1], ".")
if len(matches) > 2 {
u.Host += matches[2]
}
}
}
}
func removeEmptyPortSeparator(u *url.URL) {
if len(u.Host) > 0 {
u.Host = rxEmptyPort.ReplaceAllString(u.Host, "")
}
}

View File

@ -1,15 +0,0 @@
language: go
go:
- 1.4.x
- 1.5.x
- 1.6.x
- 1.7.x
- 1.8.x
- tip
install:
- go build .
script:
- go test -v

View File

@ -1,27 +0,0 @@
Copyright (c) 2012 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View File

@ -1,16 +0,0 @@
urlesc [![Build Status](https://travis-ci.org/PuerkitoBio/urlesc.svg?branch=master)](https://travis-ci.org/PuerkitoBio/urlesc) [![GoDoc](http://godoc.org/github.com/PuerkitoBio/urlesc?status.svg)](http://godoc.org/github.com/PuerkitoBio/urlesc)
======
Package urlesc implements query escaping as per RFC 3986.
It contains some parts of the net/url package, modified so as to allow
some reserved characters incorrectly escaped by net/url (see [issue 5684](https://github.com/golang/go/issues/5684)).
## Install
go get github.com/PuerkitoBio/urlesc
## License
Go license (BSD-3-Clause)

View File

@ -1,180 +0,0 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package urlesc implements query escaping as per RFC 3986.
// It contains some parts of the net/url package, modified so as to allow
// some reserved characters incorrectly escaped by net/url.
// See https://github.com/golang/go/issues/5684
package urlesc
import (
"bytes"
"net/url"
"strings"
)
type encoding int
const (
encodePath encoding = 1 + iota
encodeUserPassword
encodeQueryComponent
encodeFragment
)
// Return true if the specified character should be escaped when
// appearing in a URL string, according to RFC 3986.
func shouldEscape(c byte, mode encoding) bool {
// §2.3 Unreserved characters (alphanum)
if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' {
return false
}
switch c {
case '-', '.', '_', '~': // §2.3 Unreserved characters (mark)
return false
// §2.2 Reserved characters (reserved)
case ':', '/', '?', '#', '[', ']', '@', // gen-delims
'!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=': // sub-delims
// Different sections of the URL allow a few of
// the reserved characters to appear unescaped.
switch mode {
case encodePath: // §3.3
// The RFC allows sub-delims and : @.
// '/', '[' and ']' can be used to assign meaning to individual path
// segments. This package only manipulates the path as a whole,
// so we allow those as well. That leaves only ? and # to escape.
return c == '?' || c == '#'
case encodeUserPassword: // §3.2.1
// The RFC allows : and sub-delims in
// userinfo. The parsing of userinfo treats ':' as special so we must escape
// all the gen-delims.
return c == ':' || c == '/' || c == '?' || c == '#' || c == '[' || c == ']' || c == '@'
case encodeQueryComponent: // §3.4
// The RFC allows / and ?.
return c != '/' && c != '?'
case encodeFragment: // §4.1
// The RFC text is silent but the grammar allows
// everything, so escape nothing but #
return c == '#'
}
}
// Everything else must be escaped.
return true
}
// QueryEscape escapes the string so it can be safely placed
// inside a URL query.
func QueryEscape(s string) string {
return escape(s, encodeQueryComponent)
}
func escape(s string, mode encoding) string {
spaceCount, hexCount := 0, 0
for i := 0; i < len(s); i++ {
c := s[i]
if shouldEscape(c, mode) {
if c == ' ' && mode == encodeQueryComponent {
spaceCount++
} else {
hexCount++
}
}
}
if spaceCount == 0 && hexCount == 0 {
return s
}
t := make([]byte, len(s)+2*hexCount)
j := 0
for i := 0; i < len(s); i++ {
switch c := s[i]; {
case c == ' ' && mode == encodeQueryComponent:
t[j] = '+'
j++
case shouldEscape(c, mode):
t[j] = '%'
t[j+1] = "0123456789ABCDEF"[c>>4]
t[j+2] = "0123456789ABCDEF"[c&15]
j += 3
default:
t[j] = s[i]
j++
}
}
return string(t)
}
var uiReplacer = strings.NewReplacer(
"%21", "!",
"%27", "'",
"%28", "(",
"%29", ")",
"%2A", "*",
)
// unescapeUserinfo unescapes some characters that need not to be escaped as per RFC3986.
func unescapeUserinfo(s string) string {
return uiReplacer.Replace(s)
}
// Escape reassembles the URL into a valid URL string.
// The general form of the result is one of:
//
// scheme:opaque
// scheme://userinfo@host/path?query#fragment
//
// If u.Opaque is non-empty, String uses the first form;
// otherwise it uses the second form.
//
// In the second form, the following rules apply:
// - if u.Scheme is empty, scheme: is omitted.
// - if u.User is nil, userinfo@ is omitted.
// - if u.Host is empty, host/ is omitted.
// - if u.Scheme and u.Host are empty and u.User is nil,
// the entire scheme://userinfo@host/ is omitted.
// - if u.Host is non-empty and u.Path begins with a /,
// the form host/path does not add its own /.
// - if u.RawQuery is empty, ?query is omitted.
// - if u.Fragment is empty, #fragment is omitted.
func Escape(u *url.URL) string {
var buf bytes.Buffer
if u.Scheme != "" {
buf.WriteString(u.Scheme)
buf.WriteByte(':')
}
if u.Opaque != "" {
buf.WriteString(u.Opaque)
} else {
if u.Scheme != "" || u.Host != "" || u.User != nil {
buf.WriteString("//")
if ui := u.User; ui != nil {
buf.WriteString(unescapeUserinfo(ui.String()))
buf.WriteByte('@')
}
if h := u.Host; h != "" {
buf.WriteString(h)
}
}
if u.Path != "" && u.Path[0] != '/' && u.Host != "" {
buf.WriteByte('/')
}
buf.WriteString(escape(u.Path, encodePath))
}
if u.RawQuery != "" {
buf.WriteByte('?')
buf.WriteString(u.RawQuery)
}
if u.Fragment != "" {
buf.WriteByte('#')
buf.WriteString(escape(u.Fragment, encodeFragment))
}
return buf.String()
}

View File

@ -1,3 +0,0 @@
.fuzz/
*.zip

View File

@ -1,10 +0,0 @@
language: go
go:
- 1.4.3
- 1.5.4
- 1.6.4
- 1.7.6
- 1.8.3
script: make check

View File

@ -1,26 +0,0 @@
Copyright (c) 2017 Ryan Armstrong. All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors
may be used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View File

@ -1,18 +0,0 @@
PACKAGE = github.com/cavaliercoder/go-cpio
all: check
check:
go test -v
cpio-fuzz.zip: *.go
go-fuzz-build $(PACKAGE)
fuzz: cpio-fuzz.zip
go-fuzz -bin=./cpio-fuzz.zip -workdir=.fuzz/
clean-fuzz:
rm -rf cpio-fuzz.zip .fuzz/crashers/* .fuzz/suppressions/*
.PHONY: all check

View File

@ -1,62 +0,0 @@
# go-cpio [![GoDoc](https://godoc.org/github.com/cavaliercoder/go-cpio?status.svg)](https://godoc.org/github.com/cavaliercoder/go-cpio) [![Build Status](https://travis-ci.org/cavaliercoder/go-cpio.svg?branch=master)](https://travis-ci.org/cavaliercoder/go-cpio) [![Go Report Card](https://goreportcard.com/badge/github.com/cavaliercoder/go-cpio)](https://goreportcard.com/report/github.com/cavaliercoder/go-cpio)
This package provides a Go native implementation of the CPIO archive file
format.
Currently, only the SVR4 (New ASCII) format is supported, both with and without
checksums.
```go
// Create a buffer to write our archive to.
buf := new(bytes.Buffer)
// Create a new cpio archive.
w := cpio.NewWriter(buf)
// Add some files to the archive.
var files = []struct {
Name, Body string
}{
{"readme.txt", "This archive contains some text files."},
{"gopher.txt", "Gopher names:\nGeorge\nGeoffrey\nGonzo"},
{"todo.txt", "Get animal handling license."},
}
for _, file := range files {
hdr := &cpio.Header{
Name: file.Name,
Mode: 0600,
Size: int64(len(file.Body)),
}
if err := w.WriteHeader(hdr); err != nil {
log.Fatalln(err)
}
if _, err := w.Write([]byte(file.Body)); err != nil {
log.Fatalln(err)
}
}
// Make sure to check the error on Close.
if err := w.Close(); err != nil {
log.Fatalln(err)
}
// Open the cpio archive for reading.
b := bytes.NewReader(buf.Bytes())
r := cpio.NewReader(b)
// Iterate through the files in the archive.
for {
hdr, err := r.Next()
if err == io.EOF {
// end of cpio archive
break
}
if err != nil {
log.Fatalln(err)
}
fmt.Printf("Contents of %s:\n", hdr.Name)
if _, err := io.Copy(os.Stdout, r); err != nil {
log.Fatalln(err)
}
fmt.Println()
}
```

View File

@ -1,8 +0,0 @@
/*
Package cpio implements access to CPIO archives. Currently, only the SVR4 (New
ASCII) format is supported, both with and without checksums.
References:
https://www.freebsd.org/cgi/man.cgi?query=cpio&sektion=5
*/
package cpio

View File

@ -1,75 +0,0 @@
package cpio
import (
"os"
"path"
"time"
)
// headerFileInfo implements os.FileInfo.
type headerFileInfo struct {
h *Header
}
// Name returns the base name of the file.
func (fi headerFileInfo) Name() string {
if fi.IsDir() {
return path.Base(path.Clean(fi.h.Name))
}
return path.Base(fi.h.Name)
}
func (fi headerFileInfo) Size() int64 { return fi.h.Size }
func (fi headerFileInfo) IsDir() bool { return fi.Mode().IsDir() }
func (fi headerFileInfo) ModTime() time.Time { return fi.h.ModTime }
func (fi headerFileInfo) Sys() interface{} { return fi.h }
func (fi headerFileInfo) Mode() (mode os.FileMode) {
// Set file permission bits.
mode = os.FileMode(fi.h.Mode).Perm()
// Set setuid, setgid and sticky bits.
if fi.h.Mode&ModeSetuid != 0 {
// setuid
mode |= os.ModeSetuid
}
if fi.h.Mode&ModeSetgid != 0 {
// setgid
mode |= os.ModeSetgid
}
if fi.h.Mode&ModeSticky != 0 {
// sticky
mode |= os.ModeSticky
}
// Set file mode bits.
// clear perm, setuid, setgid and sticky bits.
m := os.FileMode(fi.h.Mode) & 0170000
if m == ModeDir {
// directory
mode |= os.ModeDir
}
if m == ModeNamedPipe {
// named pipe (FIFO)
mode |= os.ModeNamedPipe
}
if m == ModeSymlink {
// symbolic link
mode |= os.ModeSymlink
}
if m == ModeDevice {
// device file
mode |= os.ModeDevice
}
if m == ModeCharDevice {
// Unix character device
mode |= os.ModeDevice
mode |= os.ModeCharDevice
}
if m == ModeSocket {
// Unix domain socket
mode |= os.ModeSocket
}
return mode
}

View File

@ -1,35 +0,0 @@
// +build gofuzz
package cpio
import "bytes"
import "io"
// Fuzz tests the parsing and error handling of random byte arrays using
// https://github.com/dvyukov/go-fuzz.
func Fuzz(data []byte) int {
r := NewReader(bytes.NewReader(data))
h := NewHash()
for {
hdr, err := r.Next()
if err != nil {
if hdr != nil {
panic("hdr != nil on error")
}
if err == io.EOF {
// everything worked with random input... interesting
return 1
}
// error returned for random input. Good!
return -1
}
// hash file
h.Reset()
io.CopyN(h, r, hdr.Size)
h.Sum32()
// convert file header
FileInfoHeader(hdr.FileInfo())
}
}

View File

@ -1,45 +0,0 @@
package cpio
import (
"encoding/binary"
"hash"
)
type digest struct {
sum uint32
}
// NewHash returns a new hash.Hash32 computing the SVR4 checksum.
func NewHash() hash.Hash32 {
return &digest{}
}
func (d *digest) Write(p []byte) (n int, err error) {
for _, b := range p {
d.sum += uint32(b & 0xFF)
}
return len(p), nil
}
func (d *digest) Sum(b []byte) []byte {
out := [4]byte{}
binary.LittleEndian.PutUint32(out[:], d.sum)
return append(b, out[:]...)
}
func (d *digest) Sum32() uint32 {
return d.sum
}
func (d *digest) Reset() {
d.sum = 0
}
func (d *digest) Size() int {
return 4
}
func (d *digest) BlockSize() int {
return 1
}

View File

@ -1,153 +0,0 @@
package cpio
import (
"errors"
"fmt"
"os"
"time"
)
// Mode constants from the cpio spec.
const (
ModeSetuid = 04000 // Set uid
ModeSetgid = 02000 // Set gid
ModeSticky = 01000 // Save text (sticky bit)
ModeDir = 040000 // Directory
ModeNamedPipe = 010000 // FIFO
ModeRegular = 0100000 // Regular file
ModeSymlink = 0120000 // Symbolic link
ModeDevice = 060000 // Block special file
ModeCharDevice = 020000 // Character special file
ModeSocket = 0140000 // Socket
ModeType = 0170000 // Mask for the type bits
ModePerm = 0777 // Unix permission bits
)
const (
// headerEOF is the value of the filename of the last header in a CPIO archive.
headerEOF = "TRAILER!!!"
)
var (
ErrHeader = errors.New("cpio: invalid cpio header")
)
// A FileMode represents a file's mode and permission bits.
type FileMode int64
func (m FileMode) String() string {
return fmt.Sprintf("%#o", m)
}
// IsDir reports whether m describes a directory. That is, it tests for the
// ModeDir bit being set in m.
func (m FileMode) IsDir() bool {
return m&ModeDir != 0
}
// IsRegular reports whether m describes a regular file. That is, it tests for
// the ModeRegular bit being set in m.
func (m FileMode) IsRegular() bool {
return m&^ModePerm == ModeRegular
}
// Perm returns the Unix permission bits in m.
func (m FileMode) Perm() FileMode {
return m & ModePerm
}
// Checksum is the sum of all bytes in the file data. This sum is computed
// treating all bytes as unsigned values and using unsigned arithmetic. Only
// the least-significant 32 bits of the sum are stored. Use NewHash to compute
// the actual checksum of an archived file.
type Checksum uint32
func (c Checksum) String() string {
return fmt.Sprintf("%08X", uint32(c))
}
// A Header represents a single header in a CPIO archive.
type Header struct {
DeviceID int
Inode int64 // inode number
Mode FileMode // permission and mode bits
UID int // user id of the owner
GID int // group id of the owner
Links int // number of inbound links
ModTime time.Time // modified time
Size int64 // size in bytes
Name string // filename
Linkname string // target name of link
Checksum Checksum // computed checksum
pad int64 // bytes to pad before next header
}
// FileInfo returns an os.FileInfo for the Header.
func (h *Header) FileInfo() os.FileInfo {
return headerFileInfo{h}
}
// FileInfoHeader creates a partially-populated Header from fi.
// If fi describes a symlink, FileInfoHeader records link as the link target.
// If fi describes a directory, a slash is appended to the name.
// Because os.FileInfo's Name method returns only the base name of
// the file it describes, it may be necessary to modify the Name field
// of the returned header to provide the full path name of the file.
func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) {
if fi == nil {
return nil, errors.New("cpio: FileInfo is nil")
}
if sys, ok := fi.Sys().(*Header); ok {
// This FileInfo came from a Header (not the OS). Return a copy of the
// original Header.
h := &Header{}
*h = *sys
return h, nil
}
fm := fi.Mode()
h := &Header{
Name: fi.Name(),
Mode: FileMode(fi.Mode().Perm()), // or'd with Mode* constants later
ModTime: fi.ModTime(),
Size: fi.Size(),
}
switch {
case fm.IsRegular():
h.Mode |= ModeRegular
case fi.IsDir():
h.Mode |= ModeDir
h.Name += "/"
h.Size = 0
case fm&os.ModeSymlink != 0:
h.Mode |= ModeSymlink
h.Linkname = link
case fm&os.ModeDevice != 0:
if fm&os.ModeCharDevice != 0 {
h.Mode |= ModeCharDevice
} else {
h.Mode |= ModeDevice
}
case fm&os.ModeNamedPipe != 0:
h.Mode |= ModeNamedPipe
case fm&os.ModeSocket != 0:
h.Mode |= ModeSocket
default:
return nil, fmt.Errorf("cpio: unknown file mode %v", fm)
}
if fm&os.ModeSetuid != 0 {
h.Mode |= ModeSetuid
}
if fm&os.ModeSetgid != 0 {
h.Mode |= ModeSetgid
}
if fm&os.ModeSticky != 0 {
h.Mode |= ModeSticky
}
return h, nil
}

View File

@ -1,72 +0,0 @@
package cpio
import (
"io"
"io/ioutil"
)
// A Reader provides sequential access to the contents of a CPIO archive. A CPIO
// archive consists of a sequence of files. The Next method advances to the next
// file in the archive (including the first), and then it can be treated as an
// io.Reader to access the file's data.
type Reader struct {
r io.Reader // underlying file reader
hdr *Header // current Header
eof int64 // bytes until the end of the current file
}
// NewReader creates a new Reader reading from r.
func NewReader(r io.Reader) *Reader {
return &Reader{
r: r,
}
}
// Read reads from the current entry in the CPIO archive. It returns 0, io.EOF
// when it reaches the end of that entry, until Next is called to advance to the
// next entry.
func (r *Reader) Read(p []byte) (n int, err error) {
if r.hdr == nil || r.eof == 0 {
return 0, io.EOF
}
rn := len(p)
if r.eof < int64(rn) {
rn = int(r.eof)
}
n, err = r.r.Read(p[0:rn])
r.eof -= int64(n)
return
}
// Next advances to the next entry in the CPIO archive.
// io.EOF is returned at the end of the input.
func (r *Reader) Next() (*Header, error) {
if r.hdr == nil {
return r.next()
}
skp := r.eof + r.hdr.pad
if skp > 0 {
_, err := io.CopyN(ioutil.Discard, r.r, skp)
if err != nil {
return nil, err
}
}
return r.next()
}
func (r *Reader) next() (*Header, error) {
r.eof = 0
hdr, err := readHeader(r.r)
if err != nil {
return nil, err
}
r.hdr = hdr
r.eof = hdr.Size
return hdr, nil
}
// ReadHeader creates a new Header, reading from r.
func readHeader(r io.Reader) (*Header, error) {
// currently only SVR4 format is supported
return readSVR4Header(r)
}

Some files were not shown because too many files have changed in this diff Show More