diff --git a/.gitignore b/.gitignore index 7090e23..6c7cd59 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ venv __pycache__ clouds*.yaml *clouds.yaml - +*.yaml_????-??-??_??-??-?? diff --git a/README.md b/README.md index 98d28ad..0dba608 100644 --- a/README.md +++ b/README.md @@ -26,35 +26,26 @@ basis for later automation. # Usage ``` -$ ./openstack_workload_generator --help -usage: Create workloads on openstack installations [-h] [--log_level loglevel] [--os_cloud OS_CLOUD] - [--ansible_inventory [ANSIBLE_INVENTORY]] - [--clouds_yaml [CLOUDS_YAML]] [--wait_for_machines] - [--generate_clouds_yaml [GENERATE_CLOUDS_YAML]] - [--config CONFIG] - (--create_domains DOMAINNAME [DOMAINNAME ...] | - --delete_domains DOMAINNAME [DOMAINNAME ...]) - [--create_projects PROJECTNAME [PROJECTNAME ...] | - --delete_projects PROJECTNAME [PROJECTNAME ...]] - [--create_machines SERVERNAME [SERVERNAME ...] | - --delete_machines SERVERNAME [SERVERNAME ...]] +$ openstack_workload_generator --help +usage: Create workloads on openstack installations [-h] [--log_level loglevel] [--os_cloud OS_CLOUD] [--ansible_inventory [ANSIBLE_INVENTORY]] + [--clouds_yaml [CLOUDS_YAML]] [--wait_for_machines] [--generate_clouds_yaml [GENERATE_CLOUDS_YAML]] + [--config CONFIG] (--create_domains DOMAINNAME [DOMAINNAME ...] | --delete_domains DOMAINNAME [DOMAINNAME ...]) + [--create_projects PROJECTNAME [PROJECTNAME ...] | --delete_projects PROJECTNAME [PROJECTNAME ...]] + [--create_machines SERVERNAME [SERVERNAME ...] | --delete_machines SERVERNAME [SERVERNAME ...]] options: -h, --help show this help message and exit --log_level loglevel The loglevel - --os_cloud OS_CLOUD The openstack config to use, defaults to the value of the OS_CLOUD environment variable or - "admin" if the variable is not set + --os_cloud OS_CLOUD The openstack config to use, defaults to the value of the OS_CLOUD environment variable or "admin" if the variable is not set --ansible_inventory [ANSIBLE_INVENTORY] - Dump the created servers as an ansible inventory to the specified directory, adds a ssh - proxy jump for the hosts without a floating ip + Dump the created servers as an ansible inventory to the specified directory, adds a ssh proxy jump for the hosts without a floating ip --clouds_yaml [CLOUDS_YAML] Use a specific clouds.yaml file - --wait_for_machines Wait for every machine to be created (normally the provisioning only waits for machines - which use floating ips) + --wait_for_machines Wait for every machine to be created (normally the provisioning only waits for machines which use floating ips) --generate_clouds_yaml [GENERATE_CLOUDS_YAML] Generate a openstack clouds.yaml file - --config CONFIG The config file for environment creation, define a path to the yaml file or a subpath in - the profiles folder + --config CONFIG The config file for environment creation, define a path to the yaml file or a subpath in the profiles folder of the tool (you can overload + the search path by setting the OPENSTACK_WORKLOAD_MANAGER_PROFILES environment variable) --create_domains DOMAINNAME [DOMAINNAME ...] A list of domains to be created --delete_domains DOMAINNAME [DOMAINNAME ...] @@ -62,14 +53,43 @@ options: --create_projects PROJECTNAME [PROJECTNAME ...] A list of projects to be created in the created domains --delete_projects PROJECTNAME [PROJECTNAME ...] - A list of projects to be deleted in the created domains, all child elements are - recursively deleted + A list of projects to be deleted in the created domains, all child elements are recursively deleted --create_machines SERVERNAME [SERVERNAME ...] A list of vms to be created in the created domains --delete_machines SERVERNAME [SERVERNAME ...] A list of vms to be deleted in the created projects ``` +# Configuration + +The following cnfigurations: + +* `admin_domain_password` + * the password for the domain users which are created (User `_admin`) + * If you add "ASK_PASSWORD" as a value, the password will be asked in an interactive way +* `admin_vm_password`: + * the password for the operating system user (the username depends on the type of image you are using) + * If you add "ASK_PASSWORD" as a value, the password will be asked in an interactive way +* `vm_flavor`: + * the name of the flavor used to create virtual machines + * see `openstack flavor list` +* `vm_image`: + * the name of the image used to create virtual machines + * see `openstack image list` +* `vm_volume_size_gb`: + * the size of the persistent root volume +* `project_ipv4_subnet`: + * the network cidr of the internal network +* `*_quotas`: + * the quotas for the created projects + * execute the tool with `--log_level DEBUG` to see the configurable values +* `public_network`: + * The name of the public network which is used for floating ips +* `admin_vm_ssh_key`: + * A multiline string which ssh public keys + +``` + # Testing Scenarios ## Example usage: A minimal scenario diff --git a/src/openstack_workload_generator/__main__.py b/src/openstack_workload_generator/__main__.py index 3c73163..ba7857c 100644 --- a/src/openstack_workload_generator/__main__.py +++ b/src/openstack_workload_generator/__main__.py @@ -22,6 +22,52 @@ ) +def establish_connection(): + if args.clouds_yaml is None: + config = loader.OpenStackConfig() + else: + LOGGER.info(f"Loading connection configuration from {args.clouds_yaml}") + config = loader.OpenStackConfig(config_files=[args.clouds_yaml]) + cloud_config = config.get_one(args.os_cloud) + return Connection(config=cloud_config) + + +def generated_clouds_yaml(): + LOGGER.info(f"Creating a clouds yaml : {args.generate_clouds_yaml}") + clouds_yaml_data_new = {"clouds": clouds_yaml_data} + initial_entries = 0 + generated_entries = 0 + total_entries = 0 + if os.path.exists(args.generate_clouds_yaml): + with open(args.generate_clouds_yaml, "r") as file: + existing_data = yaml.safe_load(file) + + initial_entries = len(existing_data.get("clouds", [])) + backup_file = f"{args.generate_clouds_yaml}_{iso_timestamp()}" + logging.warning( + f"File {args.generate_clouds_yaml}, making an backup to {backup_file} and adding the new values" + ) + shutil.copy2( + args.generate_clouds_yaml, + f"{args.generate_clouds_yaml}_{iso_timestamp()}", + ) + + generated_entries = len(clouds_yaml_data_new.get("clouds", [])) + clouds_yaml_data_new = deep_merge_dict(existing_data, clouds_yaml_data_new) + total_entries = len(clouds_yaml_data_new.get("clouds", [])) + with open(args.generate_clouds_yaml, "w") as file: + yaml.dump( + clouds_yaml_data_new, + file, + default_flow_style=False, + explicit_start=True, + ) + LOGGER.info( + f"Generated {generated_entries} entries, number of entries in " + f"{args.generate_clouds_yaml} is now {total_entries} (old {initial_entries} entries)" + ) + + LOGGER = logging.getLogger() parser = argparse.ArgumentParser(prog="Create workloads on openstack installations") @@ -141,22 +187,12 @@ setup_logging(args.log_level) - -def establish_connection(): - if args.clouds_yaml is None: - config = loader.OpenStackConfig() - else: - LOGGER.info(f"Loading connection configuration from {args.clouds_yaml}") - config = loader.OpenStackConfig(config_files=[args.clouds_yaml]) - cloud_config = config.get_one(args.os_cloud) - return Connection(config=cloud_config) - - time_start = time.time() Config.load_config(args.config) Config.show_effective_config() + if args.create_domains: conn = establish_connection() workload_domains: dict[str, WorkloadGeneratorDomain] = dict() @@ -172,16 +208,18 @@ def establish_connection(): for workload_domain in workload_domains.values(): for workload_project in workload_domain.get_projects(args.create_projects): + if args.generate_clouds_yaml: + clouds_yaml_data[ + f"{workload_domain.domain_name}-{workload_project.project_name}" + ] = workload_project.get_clouds_yaml_data() + if args.create_machines: workload_project.get_and_create_machines( args.create_machines, args.wait_for_machines ) if args.ansible_inventory: workload_project.dump_inventory_hosts(args.ansible_inventory) - if args.generate_clouds_yaml: - clouds_yaml_data[ - f"{workload_domain.domain_name}-{workload_project.project_name}" - ] = workload_project.get_clouds_yaml_data() + elif args.delete_machines: for machine_obj in workload_project.get_machines( args.delete_machines @@ -189,31 +227,8 @@ def establish_connection(): machine_obj.delete_machine() if args.generate_clouds_yaml: - LOGGER.info(f"Creating a clouds yaml : {args.generate_clouds_yaml}") - clouds_yaml_data_new = {"clouds": clouds_yaml_data} - - if os.path.exists(args.generate_clouds_yaml): - with open(args.generate_clouds_yaml, "r") as file: - existing_data = yaml.safe_load(file) - backup_file = f"{args.generate_clouds_yaml}_{iso_timestamp()}" - logging.warning( - f"File {args.generate_clouds_yaml}, making an backup to {backup_file} and adding the new values" - ) - shutil.copy2( - args.generate_clouds_yaml, - f"{args.generate_clouds_yaml}_{iso_timestamp()}", - ) - clouds_yaml_data_new = deep_merge_dict( - existing_data, clouds_yaml_data_new - ) - - with open(args.generate_clouds_yaml, "w") as file: - yaml.dump( - clouds_yaml_data_new, - file, - default_flow_style=False, - explicit_start=True, - ) + generated_clouds_yaml() + sys.exit(0) elif args.delete_projects: conn = establish_connection() diff --git a/src/openstack_workload_generator/entities/domain.py b/src/openstack_workload_generator/entities/domain.py index 5ed3488..a194b4c 100644 --- a/src/openstack_workload_generator/entities/domain.py +++ b/src/openstack_workload_generator/entities/domain.py @@ -97,6 +97,7 @@ def create_and_get_projects(self, create_projects: list[str]): for project_name in create_projects: if project_name in self.workload_projects: + self.workload_projects[project_name].adapt_quota() continue project = WorkloadGeneratorProject( self.conn, project_name, self.obj, self.workload_user diff --git a/src/openstack_workload_generator/entities/helpers.py b/src/openstack_workload_generator/entities/helpers.py index d9866ca..5ded413 100644 --- a/src/openstack_workload_generator/entities/helpers.py +++ b/src/openstack_workload_generator/entities/helpers.py @@ -1,3 +1,4 @@ +import getpass import inspect import logging import os @@ -62,9 +63,14 @@ def get(key: str, regex: str = ".+", multi_line: bool = False) -> str: @staticmethod def load_config(config_file: str): - potential_profile_file = str( + + profile_path = str( os.path.realpath(os.path.dirname(os.path.realpath(__file__))) - + f"/../../../profiles/{config_file}" + + "/../../../profiles/" + ) + potential_profile_file = str( + Path(os.getenv("OPENSTACK_WORKLOAD_MANAGER_PROFILES", profile_path)) + / Path(config_file) ) if os.getenv("OPENSTACK_WORKLOAD_MANAGER_PROFILES", None): @@ -135,7 +141,9 @@ def get_number_of_floating_ips_per_project() -> int: @staticmethod def get_admin_vm_password() -> str: - return Config.get("admin_vm_password") + if Config.get("admin_vm_password").upper() == "ASK_PASSWORD": + Config._config["admin_vm_password"] = getpass.getpass("Enter the wanted admin_vm_password: ") + return Config.get("admin_vm_password", regex=r".{5,}") @staticmethod def get_vm_flavor() -> str: @@ -171,6 +179,8 @@ def get_admin_vm_ssh_key() -> str: @staticmethod def get_admin_domain_password() -> str: + if Config.get("admin_domain_password").upper() == "ASK_PASSWORD": + Config._config["admin_domain_password"] = getpass.getpass("Enter the wanted admin_domain_password: ") return Config.get("admin_domain_password", regex=r".{5,}") @staticmethod @@ -192,7 +202,9 @@ def configured_quota_names(quota_category: str) -> list[str]: @staticmethod def quota(quota_name: str, quota_category: str, default_value: int) -> int: if quota_category in Config._config: - value = Config._config.get(quota_name, default_value) + value = Config._config[quota_category].get( # type: ignore + quota_name, default_value + ) if isinstance(value, int): return value else: @@ -246,7 +258,7 @@ def setup_logging(log_level: str) -> Tuple[logging.Logger, str]: ) logger = logging.getLogger() log_file = "STDOUT" - logging.basicConfig(format=log_format_string, level=log_level) + logging.basicConfig(format=log_format_string, level=log_level.upper()) coloredlogs.DEFAULT_FIELD_STYLES["levelname"] = {"bold": True, "color": ""} coloredlogs.install(fmt=log_format_string, level=log_level.upper()) diff --git a/src/openstack_workload_generator/entities/project.py b/src/openstack_workload_generator/entities/project.py index decf5d3..7aaaa97 100644 --- a/src/openstack_workload_generator/entities/project.py +++ b/src/openstack_workload_generator/entities/project.py @@ -169,8 +169,6 @@ def _set_quota(self, quota_category: str): else: raise RuntimeError(f"Not implemented: {quota_category}") - # service_obj = getattr(self._admin_conn, api_area) - # current_quota = service_obj.get_quota_set(self.obj.id) LOGGER.debug(f"current quotas for {quota_category} : {current_quota}") new_quota = {} @@ -193,7 +191,7 @@ def _set_quota(self, quota_category: str): ) new_quota[key_name] = new_value - if len(new_quota): + if len(new_quota.keys()) > 0: set_quota_method = getattr(self._admin_conn, f"set_{api_area}_quotas") set_quota_method(self.obj.id, **new_quota) LOGGER.info(