direktil/pkg
4
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
Mikaël Cluseau 2018-06-12 20:52:20 +11:00
commit 1293dd0444
13 changed files with 1157 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md Normal file
View File

@ -0,0 +1 @@
Common go packages used in Direktil.

27
cas/cas.go Normal file
View File

@ -0,0 +1,27 @@
// Package cas provides a content-accessible storage implementation
package cas
import (
"io"
"time"
)
// Content is an item's content.
type Content interface {
io.Reader
io.Seeker
io.Closer
}
// Meta is an item's metadata.
type Meta interface {
Size() int64
ModTime() time.Time
}
// Store is a CAS store.
type Store interface {
GetOrCreate(tag, item string, create func(io.Writer) error) (content Content, meta Meta, err error)
Tags() (tags []string, err error)
Remove(tag string) error
}

112
cas/dir-store.go Normal file
View File

@ -0,0 +1,112 @@
package cas
import (
"io"
"io/ioutil"
"os"
"path/filepath"
"sync"
)
// NewDirStore create a new store backed by the given directory.
func NewDir(path string) *DirStore {
return &DirStore{
path: path,
}
}
// DirStore is a Store backed by a directory.
type DirStore struct {
path string
mutex sync.Mutex
}
var _ Store = &DirStore{}
// GetOrCreate is part of the Store interface.
func (s *DirStore) GetOrCreate(tag, item string, create func(io.Writer) error) (content Content, meta Meta, err error) {
fullPath := filepath.Join(s.path, tag, item)
f, err := os.Open(fullPath)
if err != nil {
s.mutex.Lock()
defer s.mutex.Unlock()
f, err = os.Open(fullPath)
}
if err != nil {
if !os.IsNotExist(err) {
return
}
err = os.MkdirAll(filepath.Dir(fullPath), 0700)
if err != nil {
return
}
var out *os.File
out, err = os.OpenFile(fullPath+".part", os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_EXCL, 0600)
if err != nil {
return
}
err = create(out)
out.Close()
if err != nil {
os.Remove(fullPath + ".part")
return
}
if err = os.Rename(fullPath+".part", fullPath); err != nil {
return
}
f, err = os.Open(fullPath)
if err != nil {
return
}
}
stat, err := os.Stat(fullPath)
if err != nil {
return
}
return f, stat, nil
}
// Tags is part of the Store interface.
func (s *DirStore) Tags() (tags []string, err error) {
entries, err := ioutil.ReadDir(s.path)
if err != nil {
if os.IsNotExist(err) {
return []string{}, nil
}
return
}
tags = make([]string, 0, len(entries))
for _, entry := range entries {
if !entry.IsDir() {
continue
}
name := entry.Name()
if name[0] == '.' {
continue
}
tags = append(tags, name)
}
return
}
// Remove is part of the Store interface.
func (s *DirStore) Remove(tag string) error {
return os.RemoveAll(filepath.Join(s.path, tag))
}

209
clustersconfig/clustersconfig.go Normal file
View File

@ -0,0 +1,209 @@
package clustersconfig
import (
"fmt"
"io"
"io/ioutil"
"net"
"strings"
"text/template"
yaml "gopkg.in/yaml.v2"
)
type Config struct {
Hosts []*Host
Groups []*Group
Clusters []*Cluster
Configs []*Template
StaticPods []*Template `yaml:"static_pods"`
}
func FromBytes(data []byte) (*Config, error) {
config := &Config{}
if err := yaml.Unmarshal(data, config); err != nil {
return nil, err
}
return config, nil
}
func FromFile(path string) (*Config, error) {
ba, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
return FromBytes(ba)
}
func (c *Config) Host(name string) *Host {
for _, host := range c.Hosts {
if host.Name == name {
return host
}
}
return nil
}
func (c *Config) HostByIP(ip string) *Host {
for _, host := range c.Hosts {
if host.IP == ip {
return host
}
for _, otherIP := range host.IPs {
if otherIP == ip {
return host
}
}
}
return nil
}
func (c *Config) HostByMAC(mac string) *Host {
// a bit of normalization
mac = strings.Replace(strings.ToLower(mac), "-", ":", -1)
for _, host := range c.Hosts {
if strings.ToLower(host.MAC) == mac {
return host
}
}
return nil
}
func (c *Config) Group(name string) *Group {
for _, group := range c.Groups {
if group.Name == name {
return group
}
}
return nil
}
func (c *Config) Cluster(name string) *Cluster {
for _, cluster := range c.Clusters {
if cluster.Name == name {
return cluster
}
}
return nil
}
func (c *Config) ConfigTemplate(name string) *Template {
for _, cfg := range c.Configs {
if cfg.Name == name {
return cfg
}
}
return nil
}
func (c *Config) StaticPodsTemplate(name string) *Template {
for _, s := range c.StaticPods {
if s.Name == name {
return s
}
}
return nil
}
func (c *Config) SaveTo(path string) error {
ba, err := yaml.Marshal(c)
if err != nil {
return err
}
return ioutil.WriteFile(path, ba, 0600)
}
type Template struct {
Name string
Template string
parsedTemplate *template.Template
}
func (t *Template) Execute(wr io.Writer, data interface{}, extraFuncs map[string]interface{}) error {
if t.parsedTemplate == nil {
var templateFuncs = map[string]interface{}{
"indent": func(indent, s string) (indented string) {
indented = indent + strings.Replace(s, "\n", "\n"+indent, -1)
return
},
}
for name, f := range extraFuncs {
templateFuncs[name] = f
}
tmpl, err := template.New("tmpl").
Funcs(templateFuncs).
Parse(t.Template)
if err != nil {
return err
}
t.parsedTemplate = tmpl
}
return t.parsedTemplate.Execute(wr, data)
}
// Host represents a host served by this server.
type Host struct {
Name string
MAC string
IP string
IPs []string
Cluster string
Group string
Vars Vars
}
// Group represents a group of hosts and provides their configuration.
type Group struct {
Name string
Master bool
IPXE string
Kernel string
Initrd string
Config string
StaticPods string `yaml:"static_pods"`
Versions map[string]string
Vars Vars
}
// Vars store user-defined key-values
type Vars map[string]interface{}
// Cluster represents a cluster of hosts, allowing for cluster-wide variables.
type Cluster struct {
Name string
Domain string
Subnets struct {
Services string
Pods string
}
Vars Vars
}
func (c *Cluster) KubernetesSvcIP() net.IP {
return c.NthSvcIP(1)
}
func (c *Cluster) DNSSvcIP() net.IP {
return c.NthSvcIP(2)
}
func (c *Cluster) NthSvcIP(n byte) net.IP {
_, cidr, err := net.ParseCIDR(c.Subnets.Services)
if err != nil {
panic(fmt.Errorf("Invalid services CIDR: %v", err))
}
ip := cidr.IP
ip[len(ip)-1] += n
return ip
}

129
clustersconfig/dir.go Normal file
View File

@ -0,0 +1,129 @@
package clustersconfig
import (
"io/ioutil"
"path"
"path/filepath"
"strings"
yaml "gopkg.in/yaml.v2"
)
func FromDir(dirPath string) (*Config, error) {
config := &Config{}
store := dirStore{dirPath}
load := func(dir, name string, out interface{}) error {
ba, err := store.Get(path.Join(dir, name))
if err != nil {
return err
}
return yaml.Unmarshal(ba, out)
}
// load clusters
names, err := store.List("clusters")
if err != nil {
return nil, err
}
for _, name := range names {
cluster := &Cluster{Name: name}
if err := load("clusters", name, cluster); err != nil {
return nil, err
}
config.Clusters = append(config.Clusters, cluster)
}
// load groups
names, err = store.List("groups")
if err != nil {
return nil, err
}
for _, name := range names {
o := &Group{Name: name}
if err := load("groups", name, o); err != nil {
return nil, err
}
config.Groups = append(config.Groups, o)
}
// load hosts
names, err = store.List("hosts")
if err != nil {
return nil, err
}
for _, name := range names {
o := &Host{Name: name}
if err := load("hosts", name, o); err != nil {
return nil, err
}
config.Hosts = append(config.Hosts, o)
}
// load config templates
loadTemplates := func(dir string, templates *[]*Template) error {
names, err = store.List(dir)
if err != nil {
return err
}
for _, name := range names {
ba, err := store.Get(path.Join(dir, name))
if err != nil {
return err
}
o := &Template{Name: name, Template: string(ba)}
*templates = append(*templates, o)
}
return nil
}
if err := loadTemplates("configs", &config.Configs); err != nil {
return nil, err
}
if err := loadTemplates("static-pods", &config.StaticPods); err != nil {
return nil, err
}
return config, nil
}
type dirStore struct {
path string
}
// Names is part of the kvStore interface
func (b *dirStore) List(prefix string) ([]string, error) {
files, err := filepath.Glob(filepath.Join(b.path, filepath.Join(path.Split(prefix)), "*.yaml"))
if err != nil {
return nil, err
}
names := make([]string, 0, len(files))
for _, f := range files {
f2 := strings.TrimSuffix(f, ".yaml")
f2 = filepath.Base(f2)
if f2[0] == '.' { // ignore hidden files
continue
}
names = append(names, f2)
}
return names, nil
}
// Load is part of the DataBackend interface
func (b *dirStore) Get(key string) ([]byte, error) {
return ioutil.ReadFile(filepath.Join(b.path, filepath.Join(path.Split(key))+".yaml"))
}

36
color/colors.go Normal file
View File

@ -0,0 +1,36 @@
package color
import "io"
const (
None Color = ""
Reset Color = "\033[0m"
Bold Color = "\033[1m"
Red Color = "\033[91m"
Green Color = "\033[92m"
Yellow Color = "\033[93m"
Blue Color = "\033[94m"
Magenta Color = "\033[95m"
Cyan Color = "\033[96m"
White Color = "\033[97m"
// Aligned versions (yes, I'm like that)
Red____ Color = Red
Green__ Color = Green
Yellow_ Color = Yellow
Blue___ Color = Blue
Cyan___ Color = Cyan
White__ Color = White
// Extra colors
DarkGreen Color = "\033[32m"
DarkGray Color = "\033[90m"
)
type Color string
func Write(out io.Writer, color Color, message string) {
out.Write([]byte(string(color) + message + string(Reset)))
}

102
config/config.go Normal file
View File

@ -0,0 +1,102 @@
package config
import (
"fmt"
"io/ioutil"
"os"
yaml "gopkg.in/yaml.v2"
)
// Config represent this system's configuration
type Config struct {
Vars []VarDefault
Layers []string
Modules []string
RootUser struct {
PasswordHash string `yaml:"password_hash"`
AuthorizedKeys []string `yaml:"authorized_keys"`
} `yaml:"root_user"`
Storage StorageConfig
Groups []GroupDef
Users []UserDef
Files []FileDef
Networks []NetworkDef
}
type VarDefault struct {
Name string
Default string
}
type StorageConfig struct {
UdevMatch string `yaml:"udev_match"`
RemoveVolumes []string `yaml:"remove_volumes"`
Volumes []VolumeDef
}
type VolumeDef struct {
Name string
Size string
Extents string
FS string
Mount struct {
Path string
Options string
}
}
type GroupDef struct {
Name string
Gid int
}
type UserDef struct {
Name string
Gid int
Uid int
}
type FileDef struct {
Path string
Mode os.FileMode
Content string
Secret string
}
type NetworkDef struct {
Match struct {
All bool
Name string
Ping *struct {
Source string
Target string
Count int
Timeout int
}
}
Optional bool
Script string
}
func Load(file string) (config *Config, err error) {
config = &Config{}
configData, err := ioutil.ReadFile(file)
if err != nil {
return nil, fmt.Errorf("failed to read %s: %v", file, err)
}
err = yaml.Unmarshal(configData, config)
if err != nil {
return nil, fmt.Errorf("failed to parse %s: %v", file, err)
}
return
}

52
log/compress.go Normal file
View File

@ -0,0 +1,52 @@
package log
import (
"fmt"
"io"
"os"
"path/filepath"
"github.com/ulikunitz/xz"
)
func compress(path string) {
in, err := os.Open(path)
if err != nil {
fmt.Fprintf(os.Stderr, "compress %s: failed to open: %v", path, err)
return
}
defer in.Close()
outPath := filepath.Join(filepath.Dir(path), "archives", filepath.Base(path)+".xz")
os.MkdirAll(filepath.Dir(outPath), 0700)
out, err := os.Create(outPath)
if err != nil {
fmt.Fprintf(os.Stderr, "compress %s: failed to create target: %v", path, err)
return
}
defer out.Close()
w, err := xz.NewWriter(out)
if err != nil {
fmt.Fprintf(os.Stderr, "compress %s: failed to create writer: %v", path, err)
return
}
if _, err := io.Copy(w, in); err != nil {
fmt.Fprintf(os.Stderr, "compress %s: write failed: %v", path, err)
return
}
if err := w.Close(); err != nil {
fmt.Fprintf(os.Stderr, "compress %s: close failed: %v", path, err)
return
}
if err := os.Remove(path); err != nil {
fmt.Fprintf(os.Stderr, "compress %s: failed to remove source: %v", path, err)
}
}

119
log/entry.go Normal file
View File

@ -0,0 +1,119 @@
package log
import (
"encoding/base32"
"fmt"
"io"
"time"
)
type Entry struct {
Time time.Time
Taint Taint
Data []byte
}
// WriteTo writes this log entry to w.
// Automatically appends a new line if it's not already present to
// get an easy to read log.
func (e Entry) WriteTo(w io.Writer) (n int64, err error) {
l := len(e.Data)
t := e.Time.UnixNano()
flags := byte(0)
appendNL := e.Data[len(e.Data)-1] != '\n'
if appendNL {
flags |= AppendNL
}
b := []byte{
flags,
byte(e.Taint),
byte(l >> 16 & 0xff),
byte(l >> 8 & 0xff),
byte(l >> 0 & 0xff),
byte(t >> 56 & 0xff),
byte(t >> 48 & 0xff),
byte(t >> 40 & 0xff),
byte(t >> 32 & 0xff),
byte(t >> 24 & 0xff),
byte(t >> 16 & 0xff),
byte(t >> 8 & 0xff),
byte(t >> 0 & 0xff),
}
// the binary part is b32 encoded. Obscure but still readable in text mode.
enc := base32.StdEncoding
headerLen := enc.EncodedLen(len(b))
baLen := headerLen + len(e.Data)
if appendNL {
baLen++
}
ba := make([]byte, baLen)
enc.Encode(ba, b)
copy(ba[headerLen:], e.Data)
if appendNL {
ba[baLen-1] = '\n'
}
nw, err := w.Write(ba)
return int64(nw), err
}
// ReadFrom reads the next entry from r, updating this entry.
func (e *Entry) ReadFrom(r io.Reader) (n int64, err error) {
enc := base32.StdEncoding
const L = 1 + 1 + 3 + 8
b := make([]byte, L)
ba := make([]byte, enc.EncodedLen(L))
nr, err := r.Read(ba)
if err != nil {
return int64(nr), err
}
fmt.Println(string(ba))
enc.Decode(b, ba)
fmt.Println(b)
p := 0
flags := b[p]
p++
e.Taint = Taint(b[p])
p++
l := int32(0)
for i := 0; i < 3; i++ {
l = l<<8 | int32(b[p])
p++
}
readLen := l
if flags|AppendNL != 0 {
readLen++
}
t := int64(0)
for i := 0; i < 8; i++ {
t = t<<8 | int64(b[p])
p++
}
e.Time = time.Unix(0, t)
data := make([]byte, readLen)
m, err := r.Read(data)
n += int64(m)
if err != nil {
return n, err
}
e.Data = data[:l]
return n, nil
}

38
log/entry_test.go Normal file
View File

@ -0,0 +1,38 @@
package log
import (
"bytes"
"testing"
"time"
)
func TestEncode(t *testing.T) {
l1 := Entry{time.Now(), Normal, []byte("test entry")}
l2 := Entry{}
buf := &bytes.Buffer{}
l1.WriteTo(buf)
t.Log(buf.String())
_, err := l2.ReadFrom(buf)
if err != nil {
t.Error("read error: ", err)
}
if l1.Taint != l2.Taint {
t.Errorf("wrong taint: %v != %v", l1.Taint, l2.Taint)
}
if l1.Time.UnixNano() != l2.Time.UnixNano() {
t.Errorf("wrong time: %v != %v", l1.Time, l2.Time)
}
if !bytes.Equal(l1.Data, l2.Data) {
t.Errorf("wrong data: %q != %q", string(l1.Data), string(l2.Data))
}
if l := len(buf.Bytes()); l > 0 {
t.Errorf("%d bytes not read", l)
}
}

244
log/log.go Normal file
View File

@ -0,0 +1,244 @@
package log
import (
"bufio"
"bytes"
"fmt"
"io"
"os"
"sync"
"time"
"novit.nc/direktil/src/pkg/color"
)
const (
// AppendNL indicates that a forced '\n' is added.
AppendNL byte = 1
)
var (
logs = map[string]*Log{}
mutex = sync.Mutex{}
logOutputEnabled = false
)
// Log is a log target
type Log struct {
name string
l sync.Mutex
writeToFile bool
console io.Writer
pending []Entry
out *os.File
outPath string
}
func Get(name string) *Log {
mutex.Lock()
defer mutex.Unlock()
if log, ok := logs[name]; ok {
return log
}
log := &Log{
name: name,
pending: make([]Entry, 0),
}
if logOutputEnabled {
log.enableFileOutput()
}
logs[name] = log
return log
}
// EnableFiles flushes current logs to files, and enables output to files.
func EnableFiles() {
mutex.Lock()
defer mutex.Unlock()
if logOutputEnabled {
return
}
for _, log := range logs {
// we'll let the kernel optimize, just do it all parallel
go log.enableFileOutput()
}
logOutputEnabled = true
}
// DisableFiles flushes and closes current logs files, and disables output to files.
func DisableFiles() {
mutex.Lock()
defer mutex.Unlock()
if !logOutputEnabled {
return
}
for _, log := range logs {
// we'll let the kernel optimize, just do it all parallel
go log.disableFileOutput()
}
logOutputEnabled = false
}
func (l *Log) enableFileOutput() {
l.l.Lock()
defer l.l.Unlock()
for _, e := range l.pending {
if err := l.writeEntry(e); err != nil {
l.emergencyLog(e, err)
}
}
l.writeToFile = true
}
func (l *Log) disableFileOutput() {
l.l.Lock()
defer l.l.Unlock()
if l.out != nil {
l.out.Close()
}
l.writeToFile = false
}
func (l *Log) SetConsole(console io.Writer) {
l.console = console
}
// StreamLines will copy the input line by line as log entries.
func (l *Log) StreamLines(r io.Reader) {
in := bufio.NewReader(r)
for {
line, err := in.ReadBytes('\n')
if err != nil {
if err != io.EOF {
fmt.Fprintf(os.Stderr, "log %s: read lines failed: %v\n", l.name, err)
time.Sleep(1 * time.Second)
}
return
}
l.Write(line)
}
}
// Print to this log.
func (l *Log) Print(v ...interface{}) {
fmt.Fprint(l, v...)
}
// Printf to this log.
func (l *Log) Printf(pattern string, v ...interface{}) {
fmt.Fprintf(l, pattern, v...)
}
// Taint is Print to this log with a taint.
func (l *Log) Taint(taint Taint, v ...interface{}) {
l.append(taint, []byte(fmt.Sprint(v...)))
}
// Taintf is Printf to this log with a taint.
func (l *Log) Taintf(taint Taint, pattern string, v ...interface{}) {
l.append(taint, []byte(fmt.Sprintf(pattern, v...)))
}
func (l *Log) append(taint Taint, data []byte) {
// we serialize writes
l.l.Lock()
defer l.l.Unlock()
e := Entry{
Time: time.Now(),
Taint: taint,
Data: data,
}
console := l.console
if console != nil {
buf := &bytes.Buffer{}
buf.WriteString(string(color.DarkGreen))
buf.WriteString(e.Time.Format("2006/01/02 15:04:05.000 "))
buf.WriteString(string(color.Reset))
buf.WriteString(string(e.Taint.Color()))
buf.Write(data)
if data[len(data)-1] != '\n' {
buf.Write([]byte{'\n'})
}
buf.WriteString(string(color.Reset))
buf.WriteTo(console)
}
if !l.writeToFile {
l.pending = append(l.pending, e)
// TODO if len(pending) > maxPending { pending = pending[len(pending)-underMaxPending:] }
// or use a ring
return
}
if err := l.writeEntry(e); err != nil {
l.emergencyLog(e, err)
}
}
func (l *Log) emergencyLog(entry Entry, err error) {
fmt.Fprintf(os.Stderr, "log %s: failed to write entry: %v\n -> lost entry: ", l.name, err)
entry.WriteTo(os.Stderr)
}
// Write is part of the io.Writer interface.
func (l *Log) Write(b []byte) (n int, err error) {
l.append(Normal, b)
return len(b), nil
}
func (l *Log) writeEntry(e Entry) (err error) {
path := fmt.Sprintf("/var/log/%s.%s.log",
l.name, e.Time.Truncate(time.Hour).Format(time.RFC3339))
currentPath := fmt.Sprintf("/var/log/%s.log", l.name)
if l.outPath != path {
if l.out != nil {
if err := l.out.Close(); err != nil {
fmt.Fprintf(os.Stderr, "log %s: failed to close output: %v\n", l.name, err)
}
os.Remove(currentPath)
go compress(l.outPath)
}
l.out = nil
l.outPath = ""
}
if l.out == nil {
l.out, err = os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640)
if err != nil {
return
}
l.outPath = path
os.Remove(currentPath)
if err := os.Symlink(path, currentPath); err != nil {
fmt.Fprintf(os.Stderr, "failed to symlink %s.log: %v\n", l.name, err)
}
}
_, err = e.WriteTo(l.out)
return
}

33
log/taint.go Normal file
View File

@ -0,0 +1,33 @@
package log
import (
"novit.nc/direktil/src/pkg/color"
)
const (
Normal Taint = iota
Info
Warning
Error
Fatal
OK
)
type Taint byte
func (t Taint) Color() color.Color {
switch t {
case Info:
return color.Blue
case Warning:
return color.Yellow
case Error:
return color.Red
case Fatal:
return color.Magenta
case OK:
return color.Green
default:
return color.None
}
}

55
sysfs/sysfs.go Normal file
View File

@ -0,0 +1,55 @@
package sysfs
import (
"io/ioutil"
"log"
"path/filepath"
"strings"
)
// DeviceByProperty lists the devices where a given property=value filters match.
func DeviceByProperty(class string, filters ...string) []string {
files, err := filepath.Glob("/sys/class/" + class + "/*/uevent")
if err != nil {
log.Print("list devices failed: ", err)
return nil
}
filtered := make([]string, 0)
filesLoop:
for _, file := range files {
ba, err := ioutil.ReadFile(file)
if err != nil {
log.Print("reading ", file, " failed: ", err)
continue
}
values := strings.Split(strings.TrimSpace(string(ba)), "\n")
devName := ""
for _, value := range values {
if strings.HasPrefix(value, "DEVNAME=") {
devName = value[len("DEVNAME="):]
}
}
for _, filter := range filters {
found := false
for _, value := range values {
if filter == value {
found = true
break
}
}
if !found {
continue filesLoop
}
}
filtered = append(filtered, devName)
}
return filtered
}