From 8e0491f99c3fc59500d4f5c27e1141d4f44acca7 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano <98832238+ribejara-te@users.noreply.github.com> Date: Fri, 20 Sep 2024 18:40:07 +0200 Subject: [PATCH] New features: inline secrets, custom Jinja filters! (#10) * fixed *.auto.tfvars variable autodeclaration * updated copyright year * added custom jinja2 filters support * simplified directory_remove * added inline encryption support --- src/filters.py | 35 ++++++++++++++ src/helpers.py | 72 ++++++++++++++++++++-------- src/postinit.py | 6 +-- src/preinit.py | 4 +- src/requirements.txt | 11 +++-- src/tools/cli_wrapper.py | 29 +++++++++++ src/tools/encryption_decrypt.py | 59 +++++++++++++++++++++++ src/tools/encryption_encrypt.py | 62 ++++++++++++++++++++++++ src/tools/encryption_generate_key.py | 42 ++++++++++++++++ 9 files changed, 291 insertions(+), 29 deletions(-) create mode 100644 src/filters.py create mode 100644 src/tools/cli_wrapper.py create mode 100644 src/tools/encryption_decrypt.py create mode 100644 src/tools/encryption_encrypt.py create mode 100644 src/tools/encryption_generate_key.py diff --git a/src/filters.py b/src/filters.py new file mode 100644 index 0000000..8990cfc --- /dev/null +++ b/src/filters.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Cisco Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def deepformat(value, params): + if isinstance(value, dict): + return { + deepformat(key, params): deepformat(value, params) + for key, value in value.items() + } + if isinstance(value, list): + return [ + deepformat(item, params) + for item in value + ] + if isinstance(value, str): + return value.format(**params) + return value + + +__all__ = [ + deepformat, +] diff --git a/src/helpers.py b/src/helpers.py index 187ff40..8929aab 100755 --- a/src/helpers.py +++ b/src/helpers.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2023 Cisco Systems, Inc. +# Copyright 2024 Cisco Systems, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ import fnmatch import glob import json +import os import pathlib import shutil import tempfile @@ -26,6 +27,9 @@ import jinja2 import yaml +import filters +from tools import encryption_decrypt + def directory_copy(srcpath, dstpath, ignore=[]): """Copy the contents of the dir in 'srcpath' to the dir in 'dstpath'. @@ -62,19 +66,12 @@ def directory_remove(path, keep=[]): if not path.is_dir(): return - temp = pathlib.Path(tempfile.TemporaryDirectory(dir=pathlib.Path().cwd()).name) - for item in keep: - itempath = path.joinpath(item) - if itempath.exists(): - shutil.move(itempath, temp.joinpath(item)) - - shutil.rmtree(path) - - for item in keep: - itempath = temp.joinpath(item) - if itempath.exists(): - shutil.move(itempath, path.joinpath(item)) - directory_remove(temp) + for item in path.iterdir(): + if item.name not in keep: + if item.is_dir(): + shutil.rmtree(item) + else: + item.unlink() def json_read(patterns): @@ -151,6 +148,31 @@ def hcl2_read(patterns): continue with open(path, "r") as f: data = deepmerge.always_merger.merge(data, hcl2.load(f)) + return hcl2_decrypt(data) + + +def hcl2_decrypt(data): + """Decrypts all strings in 'data'. + + Keyword arguments: + data[any]: any HCL2-sourced data structure + """ + if isinstance(data, str) and data.startswith("ENC[") and data.endswith("]"): + key_path = os.getenv("STACKS_PRIVATE_KEY_PATH") + if not key_path: + raise Exception("could not decrypt data: STACKS_PRIVATE_KEY_PATH is not set") + if not pathlib.Path(key_path).exists(): + raise Exception(f"could not decrypt data: STACKS_PRIVATE_KEY_PATH ({key_path}) does not exist") + return encryption_decrypt.main(data, key_path) + + elif isinstance(data, list): + for i in range(len(data)): + data[i] = hcl2_decrypt(data[i]) + + elif isinstance(data, dict): + for k, v in data.items(): + data[k] = hcl2_decrypt(v) + return data @@ -167,8 +189,20 @@ def jinja2_render(patterns, data): path = pathlib.Path(path) if not path.is_file(): continue - with open(path, "r") as fin: - template = jinja2.Template(fin.read()) - rendered = template.render(data) - with open(path, "w") as fout: - fout.write(rendered) + try: + with open(path, "r") as fin: + template = jinja2.Template(fin.read()) + + rendered = template.render(data | { + func.__name__: func + for func in filters.__all__ + }) + + with open(path, "w") as fout: + fout.write(rendered) + except jinja2.exceptions.UndefinedError as e: + print(f"Failure to render {path}: {e}", file=sys.stderr) + sys.exit(1) + except jinja2.exceptions.TemplateSyntaxError as e: + print(f"Failure to render {path} at line {e.lineno}, in statement {e.source}: {e}", file=sys.stderr) + sys.exit(1) diff --git a/src/postinit.py b/src/postinit.py index 136e39d..c685a2e 100755 --- a/src/postinit.py +++ b/src/postinit.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2023 Cisco Systems, Inc. +# Copyright 2024 Cisco Systems, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,10 +17,10 @@ import glob import pathlib -import helpers - import git +import helpers + # define context cwd = pathlib.Path().cwd() # current working directory diff --git a/src/preinit.py b/src/preinit.py index 45920c2..ae2e3a0 100755 --- a/src/preinit.py +++ b/src/preinit.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2023 Cisco Systems, Inc. +# Copyright 2024 Cisco Systems, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -87,7 +87,7 @@ list(variable.keys())[0] for variable in helpers.hcl2_read([workdir.joinpath("*.tf")]).get("variable", []) ] -for variable in {**variables, **helpers.hcl2_read([workdir.joinpath("*.tfvars")])}.keys(): # also autodeclare variables in *.auto.tfvars files +for variable in {**variables, **helpers.hcl2_read([workdir.joinpath("*.auto.tfvars")])}.keys(): # also autodeclare variables in *.auto.tfvars files if variable not in variables_declared: universe["variable"][variable] = {} diff --git a/src/requirements.txt b/src/requirements.txt index 1be122b..6dffa41 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,5 +1,6 @@ -deepmerge==1.0.1 -GitPython==3.1.30 -Jinja2==3.1.2 -python-hcl2==3.0.5 -PyYAML==6.0 +cryptography==43.0.1 +deepmerge==2.0 +GitPython==3.1.43 +Jinja2==3.1.4 +python-hcl2==4.3.5 +PyYAML==6.0.2 diff --git a/src/tools/cli_wrapper.py b/src/tools/cli_wrapper.py new file mode 100644 index 0000000..f62b540 --- /dev/null +++ b/src/tools/cli_wrapper.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Cisco Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def main(func): + import argparse, inspect, json + parser = argparse.ArgumentParser() + for key, value in inspect.signature(func).parameters.items(): + parser.add_argument( + f"--{key.replace('_','-')}", + action = argparse.BooleanOptionalAction if isinstance(value.default, bool) else None, + default = value.default, + required = value.default == value.empty, + ) + output = func(**vars(parser.parse_args())) + if output is not None: + print(json.dumps(output, indent=2)) diff --git a/src/tools/encryption_decrypt.py b/src/tools/encryption_decrypt.py new file mode 100644 index 0000000..1c7374c --- /dev/null +++ b/src/tools/encryption_decrypt.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Cisco Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def main(string, private_key_path): + import base64 + import cryptography.hazmat.backends + import cryptography.hazmat.primitives.asymmetric.padding + import cryptography.hazmat.primitives.hashes + import cryptography.hazmat.primitives.padding + import cryptography.hazmat.primitives.serialization + + symmetric_key_encrypted_base64, encryptor_tag_base64, init_vector_base64, string_encrypted_base64 = string.removeprefix("ENC[").removesuffix("]").split(";") + + with open(private_key_path, "rb") as key_file: + symmetric_key = cryptography.hazmat.primitives.serialization.load_pem_private_key( + key_file.read(), + password = None, + backend = cryptography.hazmat.backends.default_backend(), + ).decrypt( + base64.b64decode(symmetric_key_encrypted_base64.encode()), + cryptography.hazmat.primitives.asymmetric.padding.OAEP( + mgf = cryptography.hazmat.primitives.asymmetric.padding.MGF1(algorithm=cryptography.hazmat.primitives.hashes.SHA256()), + algorithm = cryptography.hazmat.primitives.hashes.SHA256(), + label = None, + ) + ) + + decryptor = cryptography.hazmat.primitives.ciphers.Cipher( + cryptography.hazmat.primitives.ciphers.algorithms.AES(symmetric_key), + cryptography.hazmat.primitives.ciphers.modes.GCM( + base64.b64decode(init_vector_base64.encode()), + base64.b64decode(encryptor_tag_base64.encode()), + ), + backend = cryptography.hazmat.backends.default_backend(), + ).decryptor() + padded = decryptor.update(base64.b64decode(string_encrypted_base64.encode())) + decryptor.finalize() + + unpadder = cryptography.hazmat.primitives.padding.PKCS7(128).unpadder() + unpadded = unpadder.update(padded) + unpadder.finalize() + + return unpadded.decode("utf-8") + + +if __name__ == "__main__": + import cli_wrapper + cli_wrapper.main(main) diff --git a/src/tools/encryption_encrypt.py b/src/tools/encryption_encrypt.py new file mode 100644 index 0000000..9f48f3c --- /dev/null +++ b/src/tools/encryption_encrypt.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Cisco Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def main(string, public_key_path): + import base64 + import cryptography.hazmat.backends + import cryptography.hazmat.primitives.asymmetric.padding + import cryptography.hazmat.primitives.ciphers + import cryptography.hazmat.primitives.hashes + import cryptography.hazmat.primitives.padding + import cryptography.hazmat.primitives.serialization + import os + + padder = cryptography.hazmat.primitives.padding.PKCS7(128).padder() + padded = padder.update(string.encode()) + padder.finalize() + + symmetric_key = os.urandom(32) + + init_vector = os.urandom(12) + init_vector_base64 = base64.b64encode(init_vector).decode("utf-8") + + encryptor = cryptography.hazmat.primitives.ciphers.Cipher(cryptography.hazmat.primitives.ciphers.algorithms.AES(symmetric_key), cryptography.hazmat.primitives.ciphers.modes.GCM(init_vector), backend=cryptography.hazmat.backends.default_backend()).encryptor() + + string_encrypted = encryptor.update(padded) + encryptor.finalize() + string_encrypted_base64 = base64.b64encode(string_encrypted).decode("utf-8") + + encryptor_tag_base64 = base64.b64encode(encryptor.tag).decode("utf-8") + + with open(public_key_path, "rb") as f: + public_key = cryptography.hazmat.primitives.serialization.load_pem_public_key( + f.read(), + backend = cryptography.hazmat.backends.default_backend(), + ) + + symmetric_key_encrypted_base64 = base64.b64encode(public_key.encrypt( + symmetric_key, + cryptography.hazmat.primitives.asymmetric.padding.OAEP( + mgf = cryptography.hazmat.primitives.asymmetric.padding.MGF1(algorithm=cryptography.hazmat.primitives.hashes.SHA256()), + algorithm = cryptography.hazmat.primitives.hashes.SHA256(), + label = None, + ) + )).decode("utf-8") + + return f"ENC[{symmetric_key_encrypted_base64};{encryptor_tag_base64};{init_vector_base64};{string_encrypted_base64}]" + + +if __name__ == "__main__": + import cli_wrapper + cli_wrapper.main(main) diff --git a/src/tools/encryption_generate_key.py b/src/tools/encryption_generate_key.py new file mode 100644 index 0000000..d43bc18 --- /dev/null +++ b/src/tools/encryption_generate_key.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Cisco Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def main(public_key_path, private_key_path): + import cryptography.hazmat.backends + import cryptography.hazmat.primitives.serialization + import cryptography.hazmat.primitives.asymmetric.rsa + + key = cryptography.hazmat.primitives.asymmetric.rsa.generate_private_key( + backend = cryptography.hazmat.backends.default_backend(), + key_size = 2**11, + public_exponent = 2**16+1, + ) + with open(private_key_path, "wb") as f: + f.write(key.private_bytes( + encoding = cryptography.hazmat.primitives.serialization.Encoding.PEM, + format = cryptography.hazmat.primitives.serialization.PrivateFormat.PKCS8, + encryption_algorithm = cryptography.hazmat.primitives.serialization.NoEncryption(), + )) + with open(public_key_path, "wb") as f: + f.write(key.public_key().public_bytes( + encoding = cryptography.hazmat.primitives.serialization.Encoding.PEM, + format = cryptography.hazmat.primitives.serialization.PublicFormat.SubjectPublicKeyInfo, + )) + + +if __name__ == "__main__": + import cli_wrapper + cli_wrapper.main(main)