Open Infrastructure

• To Dump ETCD https://www.npmjs.com/package/etcd-dump (Not Working Correctly)
• To Create Snapshot https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/recovery.md
• To Create a Certificate Authority to issue certificates https://coreos.com/os/docs/latest/generate-self-signed-certificates.html
• Common Name as username https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/authentication.md#using-tls-common-name

ca-config.json

{
    "signing": {
        "default": {
            "expiry": "43800h" 
        },
        "profiles": {
            "server": {
                "expiry": "43800h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth" 
                ]
            },
            "client": {
                "expiry": "43800h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "client auth" 
                ]
            },
            "peer": {
                "expiry": "43800h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth" 
                ]
            }
        }
    }
}

ca-csr.json

{
    "CN": "ungleich",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CH",
            "ST": "Glarus" 
        }
    ]
}

cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

etcd1.json

{
    "CN": "etcd1",
    "hosts": [
        "2a0a:e5c0:0:5:0:78ff:fe11:d761",
        "etcd1" 
    ],
    "key": {
        "algo": "ecdsa",
        "size": 256
    },
    "names": [
        {
            "C": "CH",
            "ST": "Glarus" 
        }
    ]
}

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server etcd1.json | cfssljson -bare etcd1

etcd2.json

{
    "CN": "etcd2",
    "hosts": [
        "2a0a:e5c0:0:5:0:78ff:fe11:d762",
        "etcd2" 
    ],
    "key": {
        "algo": "ecdsa",
        "size": 256
    },
    "names": [
        {
            "C": "CH",
            "ST": "Glarus" 
        }
    ]
}

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server etcd2.json | cfssljson -bare etcd2

etcd3.json

{
    "CN": "etcd3",
    "hosts": [
        "2a0a:e5c0:0:5:0:78ff:fe11:d763",
        "etcd3" 
    ],
    "key": {
        "algo": "ecdsa",
        "size": 256
    },
    "names": [
        {
            "C": "CH",
            "ST": "Glarus" 
        }
    ]
}

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server etcd3.json | cfssljson -bare etcd3

client.json

{
    "CN": "client",
    "hosts": [""],
    "key": {
        "algo": "ecdsa",
        "size": 256
    },
    "names": [
        {
            "C": "CH",
            "ST": "Glarus" 
        }
    ]
}

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client.json | cfssljson -bare client

root.json

{
    "CN": "root",
    "hosts": [""],
    "key": {
        "algo": "ecdsa",
        "size": 256
    },
    "names": [
        {
            "C": "CH",
            "ST": "Glarus" 
        }
    ]
}

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client root.json | cfssljson -bare root

developer.json

{
    "CN": "developer",
    "hosts": [""],
    "key": {
        "algo": "ecdsa",
        "size": 256
    },
    "names": [
        {
            "C": "CH",
            "ST": "Glarus" 
        }
    ]
}

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client developer.json | cfssljson -bare developer

To Start Fresh

rm -rf /var/lib/etcd /root/etcd-data/

# Run the following command on first node

etcd --name etcd1 --cert-file=/root/cert/etcd1.pem --key-file=/root/cert/etcd1-key.pem \
--peer-client-cert-auth --peer-trusted-ca-file=/root/cert/ca.pem \
--peer-cert-file=/root/cert/etcd1.pem --peer-key-file=/root/cert/etcd1-key.pem \
--client-cert-auth --trusted-ca-file=/root/cert/ca.pem \
--advertise-client-urls=https://[::]:2379 --listen-client-urls=https://[::]:2379 \
--initial-advertise-peer-urls=https://[::]:2380 --listen-peer-urls=https://[::]:2380 \
--initial-cluster etcd1=https://[::]:2380,etcd2=https://[2a0a:e5c0:0:5:0:78ff:fe11:d762]:2380,etcd3=https://[2a0a:e5c0:0:5:0:78ff:fe11:d763]:2380 \
--initial-cluster-state new --initial-cluster-token etcd-cluster-1 --data-dir etcd-data

# Run the following command on second node

etcd --name etcd2 --cert-file=/root/cert/etcd2.pem --key-file=/root/cert/etcd2-key.pem \
--peer-client-cert-auth --peer-trusted-ca-file=/root/cert/ca.pem \
--peer-cert-file=/root/cert/etcd2.pem --peer-key-file=/root/cert/etcd2-key.pem \
--client-cert-auth --trusted-ca-file=/root/cert/ca.pem \
--advertise-client-urls=https://[::]:2379 --listen-client-urls=https://[::]:2379 \
--initial-advertise-peer-urls=https://[::]:2380 --listen-peer-urls=https://[::]:2380 \
--initial-cluster etcd1=https://[2a0a:e5c0:0:5:0:78ff:fe11:d761]:2380,etcd2=https://[2a0a:e5c0:0:5:0:78ff:fe11:d762]:2380,etcd3=https://[2a0a:e5c0:0:5:0:78ff:fe11:d763]:2380 \
--initial-cluster-state new --initial-cluster-token etcd-cluster-1 --data-dir etcd-data

# Run the following command on third node

etcd --name etcd3 --cert-file=/root/cert/etcd3.pem --key-file=/root/cert/etcd3-key.pem \
--peer-client-cert-auth --peer-trusted-ca-file=/root/cert/ca.pem \
--peer-cert-file=/root/cert/etcd3.pem --peer-key-file=/root/cert/etcd3-key.pem \
--client-cert-auth --trusted-ca-file=/root/cert/ca.pem \
--advertise-client-urls=https://[::]:2379 --listen-client-urls=https://[::]:2379 \
--initial-advertise-peer-urls=https://[2a0a:e5c0:0:5:0:78ff:fe11:d763]:2380 --listen-peer-urls=https://[2a0a:e5c0:0:5:0:78ff:fe11:d763]:2380 \
--initial-cluster etcd1=https://[2a0a:e5c0:0:5:0:78ff:fe11:d761]:2380,etcd2=https://[2a0a:e5c0:0:5:0:78ff:fe11:d762]:2380,etcd3=https://[2a0a:e5c0:0:5:0:78ff:fe11:d763]:2380 \
--initial-cluster-state new --initial-cluster-token etcd-cluster-1 --data-dir etcd-data

Correct, permissions

chown -R etcd:etcd /var/lib/etcd/

Queries to check if things are working correctly¶

Write something

ETCDCTL_API=3 etcdctl --endpoints https://[2a0a:e5c0:0:5:0:78ff:fe11:d761]:2379 --cacert ca.pem --cert root.pem --key root-key.pem put /v1 abc

Read it

ETCDCTL_API=3 etcdctl --endpoints https://[2a0a:e5c0:0:5:0:78ff:fe11:d761]:2379 --cacert ca.pem --cert root.pem --key root-key.pem get /v1

Enable Authentication¶

• Create root user, grant it root role and Enable Authentication**
  

  ETCDCTL_API=3 etcdctl --endpoints https://[2a0a:e5c0:0:5:0:78ff:fe11:d761]:2379 --cacert ca.pem --cert root.pem --key root-key.pem user add root
  
  ETCDCTL_API=3 etcdctl --endpoints https://[2a0a:e5c0:0:5:0:78ff:fe11:d761]:2379 --cacert ca.pem --cert root.pem --key root-key.pem user grant-role root root
  
  ETCDCTL_API=3 etcdctl --endpoints https://[2a0a:e5c0:0:5:0:78ff:fe11:d761]:2379 --cacert ca.pem --cert root.pem --key root-key.pem auth enable
  

Create a non-root User e.g developer¶

ETCDCTL_API=3 etcdctl --endpoints https://[2a0a:e5c0:0:5:0:78ff:fe11:d761]:2379 --cacert ca.pem --cert root.pem --key root-key.pem role add developer

ETCDCTL_API=3 etcdctl --endpoints https://[2a0a:e5c0:0:5:0:78ff:fe11:d761]:2379 --cacert ca.pem --cert root.pem --key root-key.pem role grant-permission developer --prefix=true readwrite /v1

ETCDCTL_API=3 etcdctl --endpoints https://[2a0a:e5c0:0:5:0:78ff:fe11:d761]:2379 --cacert ca.pem --cert root.pem --key root-key.pem user add developer

ETCDCTL_API=3 etcdctl --endpoints https://[2a0a:e5c0:0:5:0:78ff:fe11:d761]:2379 --cacert ca.pem --cert root.pem --key root-key.pem user grant-role developer developer

ETCD is having some issues.

Specifically, it is saying Cluster ID mismatch I file an issue in ETCD's Github repository. https://github.com/etcd-io/etcd/issues/11263

Issue fixed.

[meow@meow-pc cert]$ ETCDCTL_API=3 etcdctl --endpoints https://[2a0a:e5c0:2:12:0:f0ff:fea9:c43a]:2379,https://[2a0a:e5c0:2:12:0:f0ff:fea9:c43d]:2379,https://[2a0a:e5c0:2:12:0:f0ff:fea9:c442]:2379 --cacert ~/Desktop/ungleich-issues/7122/cert/ca.pem --cert ~/Desktop/ungleich-issues/7122/cert/client.pem --key ~/Desktop/ungleich-issues/7122/cert/client-key.pem endpoint health
https://[2a0a:e5c0:2:12:0:f0ff:fea9:c442]:2379 is healthy: successfully committed proposal: took = 631.531912ms
https://[2a0a:e5c0:2:12:0:f0ff:fea9:c43d]:2379 is healthy: successfully committed proposal: took = 633.007889ms
https://[2a0a:e5c0:2:12:0:f0ff:fea9:c43a]:2379 is healthy: successfully committed proposal: took = 634.894405ms

• Deployed at place6
• Authentication enabled. Only clients with valid certificate issued by ungleich's private CA authority can access the etcd.
• Even finer control is employed by setting permissions for individual user to access specific keys or key's prefixes.

Only, backup is remaining.

ETCDCTL_API=3 etcdctl --endpoints https://etcd1.ungleich.ch:2379,https://etcd2.ungleich.ch:2379,https://etcd3.ungleich.ch:2379 --cacert ca.pem --cert developer.pem --key developer-key.pem endpoint health
https://etcd2.ungleich.ch:2379 is healthy: successfully committed proposal: took = 823.064847ms
https://etcd1.ungleich.ch:2379 is healthy: successfully committed proposal: took = 824.459603ms
https://etcd3.ungleich.ch:2379 is healthy: successfully committed proposal: took = 850.761864ms

Is it already in cdist?

redmine@ungleich.ch writes:

@Nico Schottelius Yes, it is in etcd-cluster branch

Dominique Roux wrote:

...

Currently there are some small problems with nftables (not loaded at boot).
Will contact alpine linux dev team

nft problem is fixed now.
Problem was: Alpine has it's own init.d script (which works ;-) ). The cdist type was already updated but the submodule was not.
The submodule is now updated too, therefore, this should not happen again in future.