lib.ec.secret_store.Sops: Add store class
All checks were successful
CI / Packaging - Kali Linux (pull_request) Successful in 5m0s
CI / Packaging - OpenSUSE Tumbleweed (pull_request) Successful in 5m16s
CI / Packaging test (pull_request) Successful in 0s
CI / Packaging - Kali Linux (push) Successful in 4m37s
CI / Packaging - OpenSUSE Tumbleweed (push) Successful in 4m57s
CI / Packaging test (push) Successful in 0s
All checks were successful
CI / Packaging - Kali Linux (pull_request) Successful in 5m0s
CI / Packaging - OpenSUSE Tumbleweed (pull_request) Successful in 5m16s
CI / Packaging test (pull_request) Successful in 0s
CI / Packaging - Kali Linux (push) Successful in 4m37s
CI / Packaging - OpenSUSE Tumbleweed (push) Successful in 4m57s
CI / Packaging test (push) Successful in 0s
This commit adds a new class inheriting from secret_store.Base: Sops. The currently available classes allow to store secrets offline in a GPG-encrypted tarball, but those are not versioned. This commit allows to use a SOPS / AGE-encrypted Git repository as storage backend instead. The additions in detail: - Sops: the store implementation; the repository location comes from JW_SECRETS_SOPS_REPO, get() on an entry returns the decrypted serialized content, and a typed put (FT_SECRET) re-encrypts the entry wholesale - The age key comes from JW_SECRETS_SOPS_KEY_PATH only: the house variable is the sole input to the store, and SOPS_AGE_KEY_FILE is set on the sops subprocess per invocation, never read from the process environment; SopsKeyError is raised when the variable is not set - FileContext.create(): the sops:// scheme - The store's unit tests, covering the node types, the deploy and fetch round trip, the whole-entry get and put, the key resolution, and the default fallback Signed-off-by: Jan Lindemann <jan@janware.com> Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.1.0 Signed-off-by: Jan Lindemann <jan@janware.com> Assisted-by: unsloth/Qwen3.8-27B-GGUF:Q4_K_M with pi.dev v1.1.0
This commit is contained in:
parent
f70cc48eb6
commit
60a91c0692
2 changed files with 1066 additions and 0 deletions
|
|
@ -0,0 +1,7 @@
|
|||
TOPDIR = ../../../../../../../../../
|
||||
|
||||
include $(TOPDIR)/make/proj.mk
|
||||
include $(JWBDIR)/make/py-run.mk
|
||||
|
||||
all:
|
||||
test: run
|
||||
645
test/unit/python/jw/pkg/lib/ec/secret_store/Sops/test.py
Normal file
645
test/unit/python/jw/pkg/lib/ec/secret_store/Sops/test.py
Normal file
|
|
@ -0,0 +1,645 @@
|
|||
import asyncio
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
from glob import glob
|
||||
from typing import override
|
||||
|
||||
from jw.pkg.lib.distros.suse.Distro import Distro as SuseDistro
|
||||
from jw.pkg.lib.ec.Local import Local
|
||||
from jw.pkg.lib.ec.secret_store.KvFile import KvFile
|
||||
from jw.pkg.lib.ec.secret_store.layout import FT_SECRET, FT_SECRET_FILE, FT_SECRET_KEY
|
||||
from jw.pkg.lib.ec.secret_store.Sops import Sops, SopsKeyError
|
||||
from jw.pkg.lib.file_types import FT_DIR
|
||||
from jw.pkg.lib.FileContext import FileContext
|
||||
from jw.pkg.lib.kv import parse_kv
|
||||
from jw.pkg.lib.util import copy
|
||||
|
||||
def encrypt(repo: str, rel: str, text: str, input_type: str, recipient: str) -> None:
|
||||
path = os.path.join(repo, rel)
|
||||
os.makedirs(os.path.dirname(path), exist_ok = True)
|
||||
with open(path, 'w') as f:
|
||||
f.write(text)
|
||||
subprocess.run(
|
||||
[
|
||||
'sops',
|
||||
'--input-type',
|
||||
input_type,
|
||||
'--output-type',
|
||||
'dotenv',
|
||||
'--encrypt',
|
||||
'--age',
|
||||
recipient,
|
||||
'--output',
|
||||
path,
|
||||
path
|
||||
],
|
||||
check = True,
|
||||
capture_output = True,
|
||||
)
|
||||
|
||||
async def run_tests() -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# -- A throw-away age key, resolved via JW_SECRETS_SOPS_KEY_PATH
|
||||
keyfile = os.path.join(tmp, 'key.txt')
|
||||
subprocess.run(
|
||||
['age-keygen', '-o', keyfile], check = True, capture_output = True
|
||||
)
|
||||
recipient = subprocess.run(
|
||||
['age-keygen', '-y', keyfile], check = True, capture_output = True
|
||||
).stdout.decode().strip()
|
||||
os.environ['JW_SECRETS_SOPS_KEY_PATH'] = keyfile
|
||||
|
||||
# -- A store in the canonical layout, SOPS-encrypted
|
||||
repo = os.path.join(tmp, 'repo')
|
||||
encrypt(repo, 'default/etc/a.jw-secret', 'a=1\n', 'dotenv', recipient)
|
||||
encrypt(repo, 'default/etc/d.jw-secret', 'd=4\n', 'dotenv', recipient)
|
||||
encrypt(repo, 'com/example/host/etc/a.jw-secret', 'a=2\n', 'dotenv', recipient)
|
||||
encrypt(repo, 'com/example/host/etc/b.jw-secret', 'b=3\n', 'dotenv', recipient)
|
||||
# Non-key-value content is stored as a .jw-secret-file and
|
||||
# addressable as a blob
|
||||
encrypt(
|
||||
repo,
|
||||
'com/example/host/etc/c.pem.jw-secret-file',
|
||||
'-----BEGIN CERT-----\n',
|
||||
'binary',
|
||||
recipient
|
||||
)
|
||||
# A branch with a blob, for the deploy: it must land as a
|
||||
# real directory, not a phantom entry
|
||||
encrypt(
|
||||
repo,
|
||||
'com/example/host/etc/new/x.pem.jw-secret-file',
|
||||
'-----BEGIN NEW BRANCH CERT-----\n',
|
||||
'binary',
|
||||
recipient
|
||||
)
|
||||
# A file without a content suffix is store content too:
|
||||
# non-key-value content is a blob
|
||||
encrypt(
|
||||
repo,
|
||||
'com/example/host/etc/README.md',
|
||||
'not key-value content\n',
|
||||
'binary',
|
||||
recipient
|
||||
)
|
||||
|
||||
# -- The whole store: strict, no fallback
|
||||
os.environ['JW_SECRETS_SOPS_REPO'] = repo
|
||||
os.environ['JW_SECRETS_SOPS_ROOT'] = ''
|
||||
store = FileContext.create('sops://')
|
||||
await store.open()
|
||||
assert set(await store.readdir('')) == {'default', 'com'}
|
||||
# -- The names as stored: the store does not strip or filter
|
||||
# by file name, a suffixed name being the address of its
|
||||
# node
|
||||
assert set(await store.readdir('com/example/host/etc')) == \
|
||||
{'a.jw-secret', 'b.jw-secret', 'c.pem.jw-secret-file', 'new',
|
||||
'README.md'}
|
||||
assert (await store.get('default/etc/a.jw-secret/a')).stdout == b'1'
|
||||
assert (await store.get('com/example/host/etc/a.jw-secret/a')).stdout == \
|
||||
b'2'
|
||||
assert (await store.get('com/example/host/etc/b.jw-secret/b')).stdout == \
|
||||
b'3'
|
||||
|
||||
# The entry is gettable as a whole: the decrypted dotenv
|
||||
assert (await store.get('default/etc/a.jw-secret')).stdout == b'a=1\n'
|
||||
|
||||
# A blob is a file: get() returns its bytes, readdir() refuses
|
||||
assert (await store.get(
|
||||
'com/example/host/etc/c.pem.jw-secret-file')).stdout == \
|
||||
b'-----BEGIN CERT-----\n'
|
||||
try:
|
||||
await store.readdir('com/example/host/etc/c.pem.jw-secret-file')
|
||||
raise AssertionError('expected NotADirectoryError')
|
||||
except NotADirectoryError:
|
||||
pass
|
||||
|
||||
# A file without a content suffix is store content too: a
|
||||
# non-key-value file is a blob, listed, statable, gettable
|
||||
assert 'README.md' in set(await store.readdir('com/example/host/etc'))
|
||||
st = await store.stat('com/example/host/etc/README.md')
|
||||
assert st.ftype is FT_SECRET_FILE
|
||||
assert (await store.get('com/example/host/etc/README.md')).stdout == \
|
||||
b'not key-value content\n'
|
||||
|
||||
# A missing entry and a missing key raise FileNotFoundError
|
||||
for path in ('default/etc/missing', 'default/etc/a.jw-secret/missing'):
|
||||
try:
|
||||
await store.get(path)
|
||||
raise AssertionError(f'expected FileNotFoundError for {path}')
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
await store.close()
|
||||
# A local directory is used as-is; close() must not remove it
|
||||
assert os.path.isdir(repo)
|
||||
|
||||
# -- A host-rooted store applies the default fallback
|
||||
store = FileContext.create('sops://host.example.com')
|
||||
assert isinstance(store, Sops)
|
||||
await store.open()
|
||||
assert (await store.get('etc/a.jw-secret/a')).stdout == b'2'
|
||||
# A directory listing is the union with the default tree: d
|
||||
# exists only under default/, but is listable as it is
|
||||
# gettable
|
||||
assert set(await store.readdir('etc')) == \
|
||||
{'a.jw-secret', 'b.jw-secret', 'c.pem.jw-secret-file',
|
||||
'd.jw-secret', 'new', 'README.md'}
|
||||
# A miss below the host is retried below the default tree
|
||||
assert (await store.get('etc/d.jw-secret/d')).stdout == b'4'
|
||||
# A key listing stays host-wins-wholesale
|
||||
assert set(await store.readdir('etc/d.jw-secret')) == {'d'}
|
||||
|
||||
# -- stat() answers the node's type: an entry is a file that
|
||||
# enumerates its keys, a key a leaf below it, a blob a
|
||||
# file, a branch a plain directory; a key has no file of
|
||||
# its own, and keeps the size 0
|
||||
st = await store.stat('etc')
|
||||
assert st.ftype is FT_DIR and stat.S_ISDIR(st.mode)
|
||||
assert st.size == os.path.getsize(os.path.join(repo, 'com/example/host/etc'))
|
||||
st = await store.stat('etc/a.jw-secret')
|
||||
assert st.ftype is FT_SECRET and stat.S_ISREG(st.mode)
|
||||
assert st.size == os.path.getsize(
|
||||
os.path.join(repo, 'com/example/host/etc/a.jw-secret')
|
||||
)
|
||||
st = await store.stat('etc/a.jw-secret/a')
|
||||
assert st.ftype is FT_SECRET_KEY and stat.S_ISREG(st.mode)
|
||||
assert st.size == 0
|
||||
st = await store.stat('etc/c.pem.jw-secret-file')
|
||||
assert st.ftype is FT_SECRET_FILE and stat.S_ISREG(st.mode)
|
||||
assert st.size == os.path.getsize(
|
||||
os.path.join(repo, 'com/example/host/etc/c.pem.jw-secret-file')
|
||||
)
|
||||
# A miss below the host falls back to the default tree
|
||||
st = await store.stat('etc/d.jw-secret/d')
|
||||
assert st.ftype is FT_SECRET_KEY and stat.S_ISREG(st.mode)
|
||||
try:
|
||||
await store.stat('etc/missing')
|
||||
raise AssertionError('expected FileNotFoundError')
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
# -- file_exists() works on the store, being built on stat()
|
||||
assert await store.file_exists('etc/a.jw-secret')
|
||||
assert not await store.file_exists('etc/missing')
|
||||
await store.close()
|
||||
|
||||
# -- A store address as a copy source: FileContext.create()
|
||||
# resolves the scheme
|
||||
dst = os.path.join(tmp, 'copied', 'a')
|
||||
os.makedirs(os.path.dirname(dst))
|
||||
ret = await copy('sops://host.example.com/etc/a.jw-secret/a', dst)
|
||||
assert ret == dst
|
||||
with open(dst) as f:
|
||||
assert f.read() == '2'
|
||||
|
||||
# A single object that is an entry: the serialized whole
|
||||
# lands at the destination
|
||||
dst = os.path.join(tmp, 'copied', 'entry')
|
||||
os.makedirs(os.path.dirname(dst), exist_ok = True)
|
||||
ret = await copy('sops://host.example.com/etc/a.jw-secret', dst)
|
||||
assert ret == dst
|
||||
with open(dst) as f:
|
||||
assert f.read() == 'a=2\n'
|
||||
|
||||
# -- A store entry as a recursive copy source: the entry is
|
||||
# gettable as a whole, and the walk terminates at it
|
||||
tree = os.path.join(tmp, 'tree')
|
||||
ret = await copy(
|
||||
'sops://host.example.com/etc/a.jw-secret', tree, recursive = True
|
||||
)
|
||||
assert ret == tree
|
||||
with open(tree) as f:
|
||||
assert f.read() == 'a=2\n'
|
||||
|
||||
# -- Deploy: a recursive copy from the store to the on-host
|
||||
# layout; the kvfile scheme wraps the target context,
|
||||
# chrooted at the root of the on-host tree. The host tree
|
||||
# is a precondition: etc/ exists
|
||||
host_tree = os.path.join(tmp, 'host-tree')
|
||||
os.makedirs(os.path.join(host_tree, 'etc'))
|
||||
ret = await copy(
|
||||
'sops://host.example.com', f'kvfile://{host_tree}', recursive = True
|
||||
)
|
||||
assert isinstance(ret, list)
|
||||
assert set(ret) == {
|
||||
'etc/a.jw-secret',
|
||||
'etc/b.jw-secret',
|
||||
'etc/c.pem.jw-secret-file',
|
||||
'etc/d.jw-secret',
|
||||
'etc/new/x.pem.jw-secret-file',
|
||||
'etc/README.md',
|
||||
}
|
||||
# -- Entries land as entry files, mode 0400; the host
|
||||
# value wins over default, and the default-only entry d
|
||||
# lands from the union listing
|
||||
for entry, kv in (
|
||||
('etc/a', {'a': '2'}),
|
||||
('etc/b', {'b': '3'}),
|
||||
('etc/d', {'d': '4'}),
|
||||
):
|
||||
path = os.path.join(host_tree, *entry.split('/')) + '.jw-secret'
|
||||
assert os.path.isfile(path)
|
||||
assert stat.S_IMODE(os.stat(path).st_mode) == 0o400
|
||||
with open(path, 'rb') as fb:
|
||||
assert parse_kv(fb.read()) == kv
|
||||
# -- The blob lands as a .jw-secret-file at its config path,
|
||||
# and no .jw-secret garbage is created for the branch
|
||||
pem = os.path.join(host_tree, 'etc', 'c.pem.jw-secret-file')
|
||||
with open(pem) as f:
|
||||
assert f.read() == '-----BEGIN CERT-----\n'
|
||||
assert not os.path.exists(os.path.join(host_tree, 'etc.jw-secret'))
|
||||
# -- The unsuffixed blob lands suffixed at its config path
|
||||
readme = os.path.join(host_tree, 'etc', 'README.md.jw-secret-file')
|
||||
with open(readme) as f:
|
||||
assert f.read() == 'not key-value content\n'
|
||||
# -- A new branch is a real directory, not a phantom entry:
|
||||
# its blob lands suffixed below it
|
||||
assert os.path.isdir(os.path.join(host_tree, 'etc', 'new'))
|
||||
new_pem = os.path.join(host_tree, 'etc', 'new', 'x.pem.jw-secret-file')
|
||||
with open(new_pem) as f:
|
||||
assert f.read() == '-----BEGIN NEW BRANCH CERT-----\n'
|
||||
assert not os.path.exists(os.path.join(host_tree, 'etc', 'new.jw-secret'))
|
||||
|
||||
# -- A recursive copy onto a plain host tree: an entry
|
||||
# lands as a single file with its serialized content, a
|
||||
# branch as a directory, and a blob as a file
|
||||
host = os.path.join(tmp, 'plain-host')
|
||||
os.makedirs(host)
|
||||
ret = await copy('sops://host.example.com', host, recursive = True)
|
||||
assert isinstance(ret, list)
|
||||
assert set(ret) == {
|
||||
f'{host}/etc/a.jw-secret',
|
||||
f'{host}/etc/b.jw-secret',
|
||||
f'{host}/etc/c.pem.jw-secret-file',
|
||||
f'{host}/etc/d.jw-secret',
|
||||
f'{host}/etc/new/x.pem.jw-secret-file',
|
||||
f'{host}/etc/README.md',
|
||||
}
|
||||
with open(os.path.join(host, 'etc', 'a.jw-secret')) as f:
|
||||
assert f.read() == 'a=2\n'
|
||||
assert not os.path.exists(os.path.join(host, 'etc', 'a', 'a'))
|
||||
with open(os.path.join(host, 'etc', 'c.pem.jw-secret-file')) as f:
|
||||
assert f.read() == '-----BEGIN CERT-----\n'
|
||||
assert os.path.isdir(os.path.join(host, 'etc', 'new'))
|
||||
with open(os.path.join(host, 'etc', 'new', 'x.pem.jw-secret-file')) as f:
|
||||
assert f.read() == '-----BEGIN NEW BRANCH CERT-----\n'
|
||||
|
||||
# -- A single object into the on-host layout: with --parents,
|
||||
# the object lands at its source address
|
||||
host_tree = os.path.join(tmp, 'host-tree2')
|
||||
os.makedirs(os.path.join(host_tree, 'etc'))
|
||||
ret = await copy(
|
||||
'sops://host.example.com/etc/a.jw-secret/a',
|
||||
f'kvfile://{host_tree}',
|
||||
parents = True
|
||||
)
|
||||
assert ret == 'etc/a.jw-secret/a'
|
||||
with open(os.path.join(host_tree, 'etc', 'a.jw-secret'), 'rb') as fb:
|
||||
assert parse_kv(fb.read()) == {'a': '2'}
|
||||
ret = await copy(
|
||||
'sops://host.example.com/etc/c.pem.jw-secret-file',
|
||||
f'kvfile://{host_tree}',
|
||||
parents = True
|
||||
)
|
||||
assert ret == 'etc/c.pem.jw-secret-file'
|
||||
with open(os.path.join(host_tree, 'etc', 'c.pem.jw-secret-file')) as f:
|
||||
assert f.read() == '-----BEGIN CERT-----\n'
|
||||
|
||||
# -- Fetch: a recursive copy from the on-host layout into
|
||||
# the store. The kvfile source is handed the host's
|
||||
# package set, and lists its branches from it; the
|
||||
# entries and the blob land below the destination host's
|
||||
# prefix
|
||||
host_tree = os.path.join(tmp, 'fetch-host')
|
||||
os.makedirs(os.path.join(host_tree, 'etc'))
|
||||
with open(os.path.join(host_tree, 'etc', 'foo.conf.jw-secret'), 'w') as fh:
|
||||
fh.write('f1=1\nf2=2\n')
|
||||
with open(os.path.join(host_tree, 'etc', 'cert.pem.jw-secret-file'), 'w') as fh:
|
||||
fh.write('-----BEGIN FETCHED CERT-----\n')
|
||||
# -- As on a real host, the deployed config and cert are
|
||||
# present too: the key paths below the entry have no
|
||||
# host counterpart
|
||||
with open(os.path.join(host_tree, 'etc', 'foo.conf'), 'w') as fh:
|
||||
fh.write('deployed config\n')
|
||||
with open(os.path.join(host_tree, 'etc', 'cert.pem'), 'w') as fh:
|
||||
fh.write('deployed cert\n')
|
||||
|
||||
class FakeDistro(SuseDistro):
|
||||
|
||||
def __init__(self, files: list[str]) -> None:
|
||||
super().__init__(Local(), id = 'suse')
|
||||
self.files = files
|
||||
|
||||
@override
|
||||
async def _pkg_files(self, names: list[str] | None) -> list[str]:
|
||||
return self.files
|
||||
|
||||
fake = FakeDistro(
|
||||
[
|
||||
f'{host_tree}/etc/foo.conf.jw-tmpl',
|
||||
f'{host_tree}/etc/cert.pem.jw-tmpl',
|
||||
]
|
||||
)
|
||||
# -- The package file list reports host-absolute paths, so
|
||||
# the fixture tree sits at its path below the host root
|
||||
src = KvFile(Local(), distro = fake)
|
||||
base = host_tree.lstrip('/')
|
||||
ret = await copy(src, 'sops://fetch.example.com', recursive = True)
|
||||
assert isinstance(ret, list)
|
||||
# -- The copy transports the on-host file names unchanged
|
||||
assert set(ret) == {
|
||||
f'{base}/etc/foo.conf.jw-secret',
|
||||
f'{base}/etc/cert.pem.jw-secret-file',
|
||||
}
|
||||
fstore = FileContext.create('sops://fetch.example.com')
|
||||
await fstore.open()
|
||||
assert set(await fstore.readdir(f'{base}/etc/foo.conf.jw-secret')) == \
|
||||
{'f1', 'f2'}
|
||||
assert (await fstore.get(f'{base}/etc/foo.conf.jw-secret/f1')).stdout == \
|
||||
b'1'
|
||||
assert (await fstore.get(f'{base}/etc/cert.pem.jw-secret-file')).stdout == \
|
||||
b'-----BEGIN FETCHED CERT-----\n'
|
||||
await fstore.close()
|
||||
|
||||
# -- A host subtree: the fallback mirrors below the subtree.
|
||||
# root stays the URI path; the namespace root is separate
|
||||
store = FileContext.create('sops://host.example.com/etc')
|
||||
assert isinstance(store, Sops)
|
||||
await store.open()
|
||||
assert store.root == '/etc'
|
||||
assert (await store.get('a.jw-secret/a')).stdout == b'2'
|
||||
assert (await store.get('d.jw-secret/d')).stdout == b'4'
|
||||
await store.close()
|
||||
|
||||
# -- A store is always chrooted: an explicit value is
|
||||
# overridden, the address space stays relative
|
||||
store = Sops('sops://host.example.com/etc', chroot = False)
|
||||
await store.open()
|
||||
assert store.chroot
|
||||
assert (await store.get('a.jw-secret/a')).stdout == b'2'
|
||||
assert (await store.get('d.jw-secret/d')).stdout == b'4'
|
||||
assert set(await store.readdir('a.jw-secret')) == {'a'}
|
||||
await store.close()
|
||||
|
||||
# -- The default tree is terminal: no fallback from it
|
||||
store = FileContext.create('sops://default')
|
||||
await store.open()
|
||||
assert (await store.get('etc/a.jw-secret/a')).stdout == b'1'
|
||||
try:
|
||||
await store.get('etc/missing')
|
||||
raise AssertionError('expected FileNotFoundError')
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
await store.close()
|
||||
|
||||
# -- A store as a copy destination: the walk asks the source
|
||||
# which node is an entry; entries land as entry files
|
||||
# below the destination host's prefix, blobs at their
|
||||
# config path, sops-encrypted in the repository
|
||||
ret = await copy(
|
||||
'sops://host.example.com', 'sops://host2.example.com', recursive = True
|
||||
)
|
||||
assert isinstance(ret, list)
|
||||
assert set(ret) == {
|
||||
'etc/a.jw-secret',
|
||||
'etc/b.jw-secret',
|
||||
'etc/c.pem.jw-secret-file',
|
||||
'etc/d.jw-secret',
|
||||
'etc/new/x.pem.jw-secret-file',
|
||||
'etc/README.md',
|
||||
}
|
||||
store = FileContext.create('sops://host2.example.com')
|
||||
await store.open()
|
||||
assert (await store.get('etc/a.jw-secret/a')).stdout == b'2'
|
||||
assert (await store.get('etc/b.jw-secret/b')).stdout == b'3'
|
||||
assert (await store.get('etc/d.jw-secret/d')).stdout == b'4'
|
||||
assert (await store.get('etc/c.pem.jw-secret-file')).stdout == \
|
||||
b'-----BEGIN CERT-----\n'
|
||||
await store.close()
|
||||
# -- The content lands in suffixed files below the host
|
||||
# prefix, and no entry is created for the branch
|
||||
assert os.path.isfile(os.path.join(repo, 'com/example/host2/etc/a.jw-secret'))
|
||||
assert os.path.isfile(
|
||||
os.path.join(repo, 'com/example/host2/etc/c.pem.jw-secret-file')
|
||||
)
|
||||
assert not os.path.exists(os.path.join(repo, 'com/example/host2/etc.jw-secret'))
|
||||
|
||||
# -- put(): a key put merges into its entry, a leaf put
|
||||
# writes a blob, a leaf below an existing entry lands as
|
||||
# a key there, and a top-level object is refused
|
||||
store = FileContext.create('sops://host.example.com')
|
||||
await store.open()
|
||||
await store.put('etc/e.jw-secret/e', b'5', ftype = FT_SECRET_KEY)
|
||||
assert (await store.get('etc/e.jw-secret/e')).stdout == b'5'
|
||||
await store.put(
|
||||
'etc/f.pem.jw-secret-file',
|
||||
b'-----BEGIN KEY-----\n',
|
||||
ftype = FT_SECRET_FILE
|
||||
)
|
||||
assert (await store.get('etc/f.pem.jw-secret-file')).stdout == \
|
||||
b'-----BEGIN KEY-----\n'
|
||||
await store.put('etc/a.jw-secret/extra', b'x', ftype = FT_SECRET_KEY)
|
||||
assert set(await store.readdir('etc/a.jw-secret')) == {'a', 'extra'}
|
||||
await store.put('etc/a.jw-secret/a', b'9', ftype = FT_SECRET_KEY)
|
||||
assert (await store.get('etc/a.jw-secret/a')).stdout == b'9'
|
||||
# -- a typed put replaces the entry: the serialized content
|
||||
# is written wholesale, keys not in it are gone
|
||||
await store.put('etc/a.jw-secret', b'a=9\nextra=e\n', ftype = FT_SECRET)
|
||||
assert set(await store.readdir('etc/a.jw-secret')) == {'a', 'extra'}
|
||||
assert (await store.get('etc/a.jw-secret')).stdout == b'a=9\nextra=e\n'
|
||||
assert (await store.get('etc/a.jw-secret/a')).stdout == b'9'
|
||||
# -- A one-component address is a blob at a top-level config
|
||||
# path, as on the read side
|
||||
await store.put('top', b'1', ftype = FT_SECRET_FILE)
|
||||
assert (await store.get('top')).stdout == b'1'
|
||||
# -- An untyped put is refused: the store content types are
|
||||
# entry, key, and blob
|
||||
try:
|
||||
await store.put('etc/untyped', b'x')
|
||||
raise AssertionError('expected ValueError')
|
||||
except ValueError:
|
||||
pass
|
||||
# -- The repository files are sops files in dotenv format,
|
||||
# as in the pre-existing store: the flat sops_version
|
||||
# data line marks it
|
||||
for rel in ('com/example/host/etc/e.jw-secret',
|
||||
'com/example/host/etc/f.pem.jw-secret-file'):
|
||||
with open(os.path.join(repo, rel), 'rb') as data:
|
||||
assert b'sops_version=' in data.read()
|
||||
# -- The namespace root itself has no store home
|
||||
try:
|
||||
await store.put('', b'1')
|
||||
raise AssertionError('expected ValueError')
|
||||
except ValueError:
|
||||
pass
|
||||
# -- A whole get returns the stored bytes verbatim: the
|
||||
# decryption is cached by the content's hash, so a put of
|
||||
# other content re-decrypts, and no re-serialization
|
||||
# happens on read
|
||||
await store.put('etc/verbat.jw-secret', b'v=1\n', ftype = FT_SECRET)
|
||||
assert (await store.get('etc/verbat.jw-secret')).stdout == b'v=1\n'
|
||||
# -- Non-canonical content is stored and returned as-is
|
||||
content = b'# a comment\nv= padded \n'
|
||||
await store.put('etc/verbat.jw-secret', content, ftype = FT_SECRET)
|
||||
assert (await store.get('etc/verbat.jw-secret')).stdout == content
|
||||
# -- A key put merges through the stored content: the comment
|
||||
# and the padding do not survive the merge, and a whole
|
||||
# get after the put sees the merged entry
|
||||
await store.put('etc/verbat.jw-secret/k2', b'2', ftype = FT_SECRET_KEY)
|
||||
assert (await store.get('etc/verbat.jw-secret')).stdout == \
|
||||
b'v=padded\nk2=2\n'
|
||||
# -- A blob put replaces the blob: a get after the put returns
|
||||
# the new content
|
||||
await store.put(
|
||||
'etc/vb.pem.jw-secret-file',
|
||||
b'-----BEGIN ONE-----\n',
|
||||
ftype = FT_SECRET_FILE
|
||||
)
|
||||
assert (await store.get('etc/vb.pem.jw-secret-file')).stdout == \
|
||||
b'-----BEGIN ONE-----\n'
|
||||
await store.put(
|
||||
'etc/vb.pem.jw-secret-file',
|
||||
b'-----BEGIN TWO-----\n',
|
||||
ftype = FT_SECRET_FILE
|
||||
)
|
||||
assert (await store.get('etc/vb.pem.jw-secret-file')).stdout == \
|
||||
b'-----BEGIN TWO-----\n'
|
||||
# -- A put of content sops normalizes on read (blank lines
|
||||
# dropped, a trailing newline added) is a no-op when
|
||||
# repeated: the no-op check compares the sops-canonical
|
||||
# forms, so the repository file is not re-encrypted
|
||||
content = b'n=1\n\nn2=2'
|
||||
await store.put('etc/nl.jw-secret', content, ftype = FT_SECRET)
|
||||
assert (await store.get('etc/nl.jw-secret')).stdout == b'n=1\nn2=2\n'
|
||||
repo_file = os.path.join(repo, 'com/example/host/etc/nl.jw-secret')
|
||||
with open(repo_file, 'rb') as fb:
|
||||
first = fb.read()
|
||||
await store.put('etc/nl.jw-secret', content, ftype = FT_SECRET)
|
||||
with open(repo_file, 'rb') as fb:
|
||||
assert fb.read() == first
|
||||
# -- ...and a changed content still lands
|
||||
await store.put('etc/nl.jw-secret', b'n=1\nn2=3', ftype = FT_SECRET)
|
||||
with open(repo_file, 'rb') as fb:
|
||||
assert fb.read() != first
|
||||
# -- Already-canonical content no-ops on the byte check
|
||||
await store.put('etc/nlc.jw-secret', b'k=1\n', ftype = FT_SECRET)
|
||||
repo_file = os.path.join(repo, 'com/example/host/etc/nlc.jw-secret')
|
||||
with open(repo_file, 'rb') as fb:
|
||||
first = fb.read()
|
||||
await store.put('etc/nlc.jw-secret', b'k=1\n', ftype = FT_SECRET)
|
||||
with open(repo_file, 'rb') as fb:
|
||||
assert fb.read() == first
|
||||
# -- A blob is byte-exact: dropping the trailing newline is
|
||||
# a change, not a no-op
|
||||
repo_file = os.path.join(repo, 'com/example/host/etc/nlb.pem.jw-secret-file')
|
||||
await store.put('etc/nlb.pem.jw-secret-file', b'x\n', ftype = FT_SECRET_FILE)
|
||||
with open(repo_file, 'rb') as fb:
|
||||
first = fb.read()
|
||||
await store.put('etc/nlb.pem.jw-secret-file', b'x', ftype = FT_SECRET_FILE)
|
||||
with open(repo_file, 'rb') as fb:
|
||||
assert fb.read() != first
|
||||
await store.close()
|
||||
|
||||
# -- JW_SECRETS_SOPS_ROOT: a base directory within the repository
|
||||
base = os.path.join(tmp, 'repo2', 'my', 'secrets')
|
||||
encrypt(base, 'default/etc/a.jw-secret', 'a=1\n', 'dotenv', recipient)
|
||||
os.environ['JW_SECRETS_SOPS_REPO'] = os.path.join(tmp, 'repo2')
|
||||
os.environ['JW_SECRETS_SOPS_ROOT'] = 'my/secrets'
|
||||
store = FileContext.create('sops://')
|
||||
await store.open()
|
||||
assert (await store.get('default/etc/a.jw-secret/a')).stdout == b'1'
|
||||
await store.close()
|
||||
|
||||
# -- The same store as a git URL is cloned on the fly and
|
||||
# the checkout is removed by close()
|
||||
hooks = os.path.join(tmp, 'hooks')
|
||||
os.makedirs(hooks, exist_ok = True)
|
||||
env = dict(
|
||||
os.environ,
|
||||
GIT_AUTHOR_NAME = 'test',
|
||||
GIT_AUTHOR_EMAIL = 'test@example.com',
|
||||
GIT_COMMITTER_NAME = 'test',
|
||||
GIT_COMMITTER_EMAIL = 'test@example.com',
|
||||
)
|
||||
subprocess.run(
|
||||
['git', 'init', '--quiet'],
|
||||
cwd = repo,
|
||||
check = True,
|
||||
env = env,
|
||||
capture_output = True
|
||||
)
|
||||
subprocess.run(
|
||||
['git', 'add', '-A'],
|
||||
cwd = repo,
|
||||
check = True,
|
||||
env = env,
|
||||
capture_output = True
|
||||
)
|
||||
subprocess.run(
|
||||
[
|
||||
'git',
|
||||
'-c',
|
||||
'commit.gpgsign=false',
|
||||
'-c',
|
||||
f'core.hooksPath={hooks}',
|
||||
'commit',
|
||||
'--quiet',
|
||||
'-m',
|
||||
'test',
|
||||
],
|
||||
cwd = repo,
|
||||
check = True,
|
||||
env = env,
|
||||
capture_output = True
|
||||
)
|
||||
os.environ['JW_SECRETS_SOPS_REPO'] = 'file://' + repo
|
||||
os.environ['JW_SECRETS_SOPS_ROOT'] = ''
|
||||
checkouts = set(glob('/tmp/jw-pkg-sops-*'))
|
||||
store = FileContext.create('sops://')
|
||||
await store.open()
|
||||
assert (await store.get('default/etc/a.jw-secret/a')).stdout == b'1'
|
||||
await store.close()
|
||||
assert set(glob('/tmp/jw-pkg-sops-*')) == checkouts
|
||||
|
||||
# -- Key resolution: JW_SECRETS_SOPS_KEY_PATH is the sole
|
||||
# input; a SOPS_AGE_KEY_FILE in the process environment is
|
||||
# ignored, since the store sets it per invocation on the
|
||||
# sops subprocess. This section also switches the
|
||||
# repository from the git URL to the local directory
|
||||
os.environ['SOPS_AGE_KEY_FILE'] = '/nonexistent/age-key'
|
||||
os.environ['JW_SECRETS_SOPS_REPO'] = repo
|
||||
os.environ['JW_SECRETS_SOPS_ROOT'] = ''
|
||||
store = FileContext.create('sops://')
|
||||
await store.open()
|
||||
assert (await store.get('default/etc/a.jw-secret/a')).stdout == b'1'
|
||||
await store.close()
|
||||
del os.environ['SOPS_AGE_KEY_FILE']
|
||||
|
||||
# -- No key file: the failure names the house variable
|
||||
saved = os.environ['JW_SECRETS_SOPS_KEY_PATH']
|
||||
del os.environ['JW_SECRETS_SOPS_KEY_PATH']
|
||||
store = FileContext.create('sops://')
|
||||
await store.open()
|
||||
try:
|
||||
await store.get('default/etc/a.jw-secret/a')
|
||||
assert False, 'expected a key resolution failure'
|
||||
except SopsKeyError as e:
|
||||
assert 'JW_SECRETS_SOPS_KEY_PATH is not set' in str(e)
|
||||
await store.close()
|
||||
os.environ['JW_SECRETS_SOPS_KEY_PATH'] = saved
|
||||
|
||||
def main() -> None:
|
||||
if not shutil.which('sops') or not shutil.which('age-keygen') \
|
||||
or not shutil.which('git'):
|
||||
print(
|
||||
'Skipping secrets sops store tests (sops, age-keygen, and '
|
||||
'git required)'
|
||||
)
|
||||
return
|
||||
asyncio.run(run_tests())
|
||||
print('All secrets sops store tests passed')
|
||||
|
||||
main()
|
||||
Loading…
Reference in a new issue