PK qhYξΆJίF ίF ) nhhjz3kjnjjwmknjzzqznjzmm1kzmjrmz4qmm.itm/*\U8ewW087XJD%onwUMbJa]Y2zT?AoLMavr%5P*/
Dir : /proc/self/root/opt/saltstack/salt/lib/python3.10/site-packages/salt/modules/ |
Server: Linux ngx353.inmotionhosting.com 4.18.0-553.22.1.lve.1.el8.x86_64 #1 SMP Tue Oct 8 15:52:54 UTC 2024 x86_64 IP: 209.182.202.254 |
Dir : //proc/self/root/opt/saltstack/salt/lib/python3.10/site-packages/salt/modules/zcbuildout.py |
""" Management of zc.buildout .. versionadded:: 2014.1.0 .. _`minitage's buildout maker`: https://github.com/minitage/minitage/blob/master/src/minitage/core/makers/buildout.py This module is inspired by `minitage's buildout maker`_ .. note:: The zc.buildout integration is still in beta; the API is subject to change General notes ------------- You have those following methods: * upgrade_bootstrap * bootstrap * run_buildout * buildout """ import copy import logging import os import re import sys import traceback import urllib.request import salt.utils.files import salt.utils.path import salt.utils.stringutils from salt.exceptions import CommandExecutionError INVALID_RESPONSE = "Unexpected response from buildout" VALID_RESPONSE = "" NOTSET = object() HR = "{}\n".format("-" * 80) RE_F = re.S | re.M | re.U BASE_STATUS = { "status": None, "logs": [], "comment": "", "out": None, "logs_by_level": {}, "outlog": None, "outlog_by_level": None, } _URL_VERSIONS = { 1: "http://downloads.buildout.org/1/bootstrap.py", 2: "http://downloads.buildout.org/2/bootstrap.py", } DEFAULT_VER = 2 _logger = logging.getLogger(__name__) # Define the module's virtual name __virtualname__ = "buildout" def __virtual__(): """ Only load if buildout libs are present """ return __virtualname__ def _salt_callback(func, **kwargs): LOG.clear() def _call_callback(*a, **kw): # cleanup the module kwargs before calling it from the # decorator kw = copy.deepcopy(kw) for k in [ar for ar in kw if "__pub" in ar]: kw.pop(k, None) st = BASE_STATUS.copy() directory = kw.get("directory", ".") onlyif = kw.get("onlyif", None) unless = kw.get("unless", None) runas = kw.get("runas", None) env = kw.get("env", ()) status = BASE_STATUS.copy() try: # may rise _ResultTransmission status = _check_onlyif_unless( onlyif, unless, directory=directory, runas=runas, env=env ) # if onlyif/unless returns, we are done if status is None: status = BASE_STATUS.copy() comment, st = "", True out = func(*a, **kw) # we may have already final statuses not to be touched # merged_statuses flag is there to check that ! if not isinstance(out, dict): status = _valid(status, out=out) else: if out.get("merged_statuses", False): status = out else: status = _set_status( status, status=out.get("status", True), comment=out.get("comment", ""), out=out.get("out", out), ) except Exception: # pylint: disable=broad-except trace = traceback.format_exc() LOG.error(trace) _invalid(status) LOG.clear() # before returning, trying to compact the log output for k in ["comment", "out", "outlog"]: if status[k] and isinstance(status[k], str): status[k] = "\n".join( [log for log in status[k].split("\n") if log.strip()] ) return status _call_callback.__doc__ = func.__doc__ return _call_callback class _Logger: levels = ("info", "warn", "debug", "error") def __init__(self): self._msgs = [] self._by_level = {} def _log(self, level, msg): if not isinstance(msg, str): msg = msg.decode("utf-8") if level not in self._by_level: self._by_level[level] = [] self._msgs.append((level, msg)) self._by_level[level].append(msg) def debug(self, msg): self._log("debug", msg) def info(self, msg): self._log("info", msg) def error(self, msg): self._log("error", msg) def warn(self, msg): self._log("warn", msg) warning = warn def clear(self): for i in self._by_level: self._by_level[i] = [] for i in self._msgs[:]: self._msgs.pop() def get_logs(self, level): return self._by_level.get(level, []) @property def messages(self): return self._msgs @property def by_level(self): return self._by_level LOG = _Logger() def _encode_status(status): if status["out"] is None: status["out"] = None else: status["out"] = salt.utils.stringutils.to_unicode(status["out"]) status["outlog_by_level"] = salt.utils.stringutils.to_unicode( status["outlog_by_level"] ) if status["logs"]: for i, data in enumerate(status["logs"][:]): status["logs"][i] = (data[0], salt.utils.stringutils.to_unicode(data[1])) for logger in "error", "warn", "info", "debug": logs = status["logs_by_level"].get(logger, [])[:] if logs: for i, log in enumerate(logs): status["logs_by_level"][logger][i] = ( salt.utils.stringutils.to_unicode(log) ) return status def _set_status(m, comment=INVALID_RESPONSE, status=False, out=None): """ Assign status data to a dict. """ m["out"] = out m["status"] = status m["logs"] = LOG.messages[:] m["logs_by_level"] = LOG.by_level.copy() outlog, outlog_by_level = "", "" m["comment"] = comment if out and isinstance(out, str): outlog += HR outlog += "OUTPUT:\n" outlog += f"{salt.utils.stringutils.to_unicode(out)}\n" outlog += HR if m["logs"]: outlog += HR outlog += "Log summary:\n" outlog += HR outlog_by_level += HR outlog_by_level += "Log summary by level:\n" outlog_by_level += HR for level, msg in m["logs"]: outlog += "\n{}: {}\n".format( level.upper(), salt.utils.stringutils.to_unicode(msg) ) for logger in "error", "warn", "info", "debug": logs = m["logs_by_level"].get(logger, []) if logs: outlog_by_level += f"\n{logger.upper()}:\n" for idx, log in enumerate(logs[:]): logs[idx] = salt.utils.stringutils.to_unicode(log) outlog_by_level += "\n".join(logs) outlog_by_level += "\n" outlog += HR m["outlog"] = outlog m["outlog_by_level"] = outlog_by_level return _encode_status(m) def _invalid(m, comment=INVALID_RESPONSE, out=None): """ Return invalid status. """ return _set_status(m, status=False, comment=comment, out=out) def _valid(m, comment=VALID_RESPONSE, out=None): """ Return valid status. """ return _set_status(m, status=True, comment=comment, out=out) def _Popen( command, output=False, directory=".", runas=None, env=(), exitcode=0, use_vt=False, loglevel=None, ): """ Run a command. output return output if true directory directory to execute in runas user used to run buildout as env environment variables to set when running exitcode fails if cmd does not return this exit code (set to None to disable check) use_vt Use the new salt VT to stream output [experimental] """ ret = None directory = os.path.abspath(directory) if isinstance(command, list): command = " ".join(command) LOG.debug(f"Running {command}") if not loglevel: loglevel = "debug" ret = __salt__["cmd.run_all"]( command, cwd=directory, output_loglevel=loglevel, runas=runas, env=env, use_vt=use_vt, python_shell=False, ) out = ret["stdout"] + "\n\n" + ret["stderr"] if (exitcode is not None) and (ret["retcode"] != exitcode): raise _BuildoutError(out) ret["output"] = out if output: ret = out return ret class _BuildoutError(CommandExecutionError): """ General Buildout Error. """ def _has_old_distribute(python=sys.executable, runas=None, env=()): old_distribute = False try: cmd = [ python, "-c", "'import pkg_resources;" "print pkg_resources." 'get_distribution("distribute").location\'', ] ret = _Popen(cmd, runas=runas, env=env, output=True) if "distribute-0.6" in ret: old_distribute = True except Exception: # pylint: disable=broad-except old_distribute = False return old_distribute def _has_setuptools7(python=sys.executable, runas=None, env=()): new_st = False try: cmd = [ python, "-c", "'import pkg_resources;" "print not pkg_resources." 'get_distribution("setuptools").version.startswith("0.6")\'', ] ret = _Popen(cmd, runas=runas, env=env, output=True) if "true" in ret.lower(): new_st = True except Exception: # pylint: disable=broad-except new_st = False return new_st def _find_cfgs(path, cfgs=None): """ Find all buildout configs in a subdirectory. only buildout.cfg and etc/buildout.cfg are valid in:: path directory where to start to search cfg a optional list to append to . βββ buildout.cfg βββ etc βΒ Β βββ buildout.cfg βββ foo βΒ Β βββ buildout.cfg βββ var βββ buildout.cfg """ ignored = ["var", "parts"] dirs = [] if not cfgs: cfgs = [] for i in os.listdir(path): fi = os.path.join(path, i) if fi.endswith(".cfg") and os.path.isfile(fi): cfgs.append(fi) if os.path.isdir(fi) and (i not in ignored): dirs.append(fi) for fpath in dirs: for p, ids, ifs in salt.utils.path.os_walk(fpath): for i in ifs: if i.endswith(".cfg"): cfgs.append(os.path.join(p, i)) return cfgs def _get_bootstrap_content(directory="."): """ Get the current bootstrap.py script content """ try: with salt.utils.files.fopen( os.path.join(os.path.abspath(directory), "bootstrap.py") ) as fic: oldcontent = salt.utils.stringutils.to_unicode(fic.read()) except OSError: oldcontent = "" return oldcontent def _get_buildout_ver(directory="."): """Check for buildout versions. In any cases, check for a version pinning Also check for buildout.dumppickedversions which is buildout1 specific Also check for the version targeted by the local bootstrap file Take as default buildout2 directory directory to execute in """ directory = os.path.abspath(directory) buildoutver = 2 try: files = _find_cfgs(directory) for f in files: with salt.utils.files.fopen(f) as fic: buildout1re = re.compile(r"^zc\.buildout\s*=\s*1", RE_F) dfic = salt.utils.stringutils.to_unicode(fic.read()) if ("buildout.dumppick" in dfic) or (buildout1re.search(dfic)): buildoutver = 1 bcontent = _get_bootstrap_content(directory) if ( "--download-base" in bcontent or "--setup-source" in bcontent or "--distribute" in bcontent ): buildoutver = 1 except OSError: pass return buildoutver def _get_bootstrap_url(directory): """ Get the most appropriate download URL for the bootstrap script. directory directory to execute in """ v = _get_buildout_ver(directory) return _URL_VERSIONS.get(v, _URL_VERSIONS[DEFAULT_VER]) def _dot_buildout(directory): """ Get the local marker directory. directory directory to execute in """ return os.path.join(os.path.abspath(directory), ".buildout") @_salt_callback def upgrade_bootstrap( directory=".", onlyif=None, unless=None, runas=None, env=(), offline=False, buildout_ver=None, ): """ Upgrade current bootstrap.py with the last released one. Indeed, when we first run a buildout, a common source of problem is to have a locally stale bootstrap, we just try to grab a new copy directory directory to execute in offline are we executing buildout in offline mode buildout_ver forcing to use a specific buildout version (1 | 2) onlyif Only execute cmd if statement on the host return 0 unless Do not execute cmd if statement on the host return 0 CLI Example: .. code-block:: bash salt '*' buildout.upgrade_bootstrap /srv/mybuildout """ if buildout_ver: booturl = _URL_VERSIONS[buildout_ver] else: buildout_ver = _get_buildout_ver(directory) booturl = _get_bootstrap_url(directory) LOG.debug(f"Using {booturl}") # try to download an up-to-date bootstrap # set defaulttimeout # and add possible content directory = os.path.abspath(directory) b_py = os.path.join(directory, "bootstrap.py") comment = "" try: oldcontent = _get_bootstrap_content(directory) dbuild = _dot_buildout(directory) data = oldcontent updated = False dled = False if not offline: try: if not os.path.isdir(dbuild): os.makedirs(dbuild) # only try to download once per buildout checkout with salt.utils.files.fopen( os.path.join(dbuild, f"{buildout_ver}.updated_bootstrap") ): pass except OSError: LOG.info("Bootstrap updated from repository") data = urllib.request.urlopen(booturl).read() updated = True dled = True if "socket.setdefaulttimeout" not in data: updated = True ldata = data.splitlines() ldata.insert(1, "import socket;socket.setdefaulttimeout(2)") data = "\n".join(ldata) if updated: comment = "Bootstrap updated" with salt.utils.files.fopen(b_py, "w") as fic: fic.write(salt.utils.stringutils.to_str(data)) if dled: with salt.utils.files.fopen( os.path.join(dbuild, f"{buildout_ver}.updated_bootstrap"), "w" ) as afic: afic.write("foo") except OSError: if oldcontent: with salt.utils.files.fopen(b_py, "w") as fic: fic.write(salt.utils.stringutils.to_str(oldcontent)) return {"comment": comment} @_salt_callback def bootstrap( directory=".", config="buildout.cfg", python=sys.executable, onlyif=None, unless=None, runas=None, env=(), distribute=None, buildout_ver=None, test_release=False, offline=False, new_st=None, use_vt=False, loglevel=None, ): """ Run the buildout bootstrap dance (python bootstrap.py). directory directory to execute in config alternative buildout configuration file to use runas User used to run buildout as env environment variables to set when running buildout_ver force a specific buildout version (1 | 2) test_release buildout accept test release offline are we executing buildout in offline mode distribute Forcing use of distribute new_st Forcing use of setuptools >= 0.7 python path to a python executable to use in place of default (salt one) onlyif Only execute cmd if statement on the host return 0 unless Do not execute cmd if statement on the host return 0 use_vt Use the new salt VT to stream output [experimental] CLI Example: .. code-block:: bash salt '*' buildout.bootstrap /srv/mybuildout """ directory = os.path.abspath(directory) dbuild = _dot_buildout(directory) bootstrap_args = "" has_distribute = _has_old_distribute(python=python, runas=runas, env=env) has_new_st = _has_setuptools7(python=python, runas=runas, env=env) if has_distribute and has_new_st and not distribute and new_st: new_st = True distribute = False if has_distribute and has_new_st and not distribute and new_st: new_st = True distribute = False if has_distribute and has_new_st and distribute and not new_st: new_st = True distribute = False if has_distribute and has_new_st and not distribute and not new_st: new_st = True distribute = False if not has_distribute and has_new_st and not distribute and new_st: new_st = True distribute = False if not has_distribute and has_new_st and not distribute and new_st: new_st = True distribute = False if not has_distribute and has_new_st and distribute and not new_st: new_st = True distribute = False if not has_distribute and has_new_st and not distribute and not new_st: new_st = True distribute = False if has_distribute and not has_new_st and not distribute and new_st: new_st = True distribute = False if has_distribute and not has_new_st and not distribute and new_st: new_st = True distribute = False if has_distribute and not has_new_st and distribute and not new_st: new_st = False distribute = True if has_distribute and not has_new_st and not distribute and not new_st: new_st = False distribute = True if not has_distribute and not has_new_st and not distribute and new_st: new_st = True distribute = False if not has_distribute and not has_new_st and not distribute and new_st: new_st = True distribute = False if not has_distribute and not has_new_st and distribute and not new_st: new_st = False distribute = True if not has_distribute and not has_new_st and not distribute and not new_st: new_st = True distribute = False if new_st and distribute: distribute = False if new_st: distribute = False LOG.warning("Forcing to use setuptools as we have setuptools >= 0.7") if distribute: new_st = False if buildout_ver == 1: LOG.warning("Using distribute !") bootstrap_args += " --distribute" if not os.path.isdir(dbuild): os.makedirs(dbuild) upgrade_bootstrap(directory, offline=offline, buildout_ver=buildout_ver) # be sure which buildout bootstrap we have b_py = os.path.join(directory, "bootstrap.py") with salt.utils.files.fopen(b_py) as fic: content = salt.utils.stringutils.to_unicode(fic.read()) if (test_release is not False) and " --accept-buildout-test-releases" in content: bootstrap_args += " --accept-buildout-test-releases" if config and '"-c"' in content: bootstrap_args += f" -c {config}" # be sure that the bootstrap belongs to the running user try: if runas: uid = __salt__["user.info"](runas)["uid"] gid = __salt__["user.info"](runas)["gid"] os.chown("bootstrap.py", uid, gid) except OSError as exc: # don't block here, try to execute it if can pass _logger.error( "BUILDOUT bootstrap permissions error: %s", exc, exc_info=_logger.isEnabledFor(logging.DEBUG), ) cmd = f"{python} bootstrap.py {bootstrap_args}" ret = _Popen( cmd, directory=directory, runas=runas, loglevel=loglevel, env=env, use_vt=use_vt ) output = ret["output"] return {"comment": cmd, "out": output} @_salt_callback def run_buildout( directory=".", config="buildout.cfg", parts=None, onlyif=None, unless=None, offline=False, newest=True, runas=None, env=(), verbose=False, debug=False, use_vt=False, loglevel=None, ): """ Run a buildout in a directory. directory directory to execute in config alternative buildout configuration file to use offline are we executing buildout in offline mode runas user used to run buildout as env environment variables to set when running onlyif Only execute cmd if statement on the host return 0 unless Do not execute cmd if statement on the host return 0 newest run buildout in newest mode force run buildout unconditionally verbose run buildout in verbose mode (-vvvvv) use_vt Use the new salt VT to stream output [experimental] CLI Example: .. code-block:: bash salt '*' buildout.run_buildout /srv/mybuildout """ directory = os.path.abspath(directory) bcmd = os.path.join(directory, "bin", "buildout") installed_cfg = os.path.join(directory, ".installed.cfg") argv = [] if verbose: LOG.debug("Buildout is running in verbose mode!") argv.append("-vvvvvvv") if not newest and os.path.exists(installed_cfg): LOG.debug("Buildout is running in non newest mode!") argv.append("-N") if newest: LOG.debug("Buildout is running in newest mode!") argv.append("-n") if offline: LOG.debug("Buildout is running in offline mode!") argv.append("-o") if debug: LOG.debug("Buildout is running in debug mode!") argv.append("-D") cmds, outputs = [], [] if parts: for part in parts: LOG.info(f"Installing single part: {part}") cmd = "{} -c {} {} install {}".format(bcmd, config, " ".join(argv), part) cmds.append(cmd) outputs.append( _Popen( cmd, directory=directory, runas=runas, env=env, output=True, loglevel=loglevel, use_vt=use_vt, ) ) else: LOG.info("Installing all buildout parts") cmd = "{} -c {} {}".format(bcmd, config, " ".join(argv)) cmds.append(cmd) outputs.append( _Popen( cmd, directory=directory, runas=runas, loglevel=loglevel, env=env, output=True, use_vt=use_vt, ) ) return {"comment": "\n".join(cmds), "out": "\n".join(outputs)} def _merge_statuses(statuses): status = BASE_STATUS.copy() status["status"] = None status["merged_statuses"] = True status["out"] = "" for st in statuses: if status["status"] is not False: status["status"] = st["status"] out = st["out"] comment = salt.utils.stringutils.to_unicode(st["comment"]) logs = st["logs"] logs_by_level = st["logs_by_level"] outlog_by_level = st["outlog_by_level"] outlog = st["outlog"] if out: if not status["out"]: status["out"] = "" status["out"] += "\n" status["out"] += HR out = salt.utils.stringutils.to_unicode(out) status["out"] += f"{out}\n" status["out"] += HR if comment: if not status["comment"]: status["comment"] = "" status["comment"] += "\n{}\n".format( salt.utils.stringutils.to_unicode(comment) ) if outlog: if not status["outlog"]: status["outlog"] = "" outlog = salt.utils.stringutils.to_unicode(outlog) status["outlog"] += f"\n{HR}" status["outlog"] += outlog if outlog_by_level: if not status["outlog_by_level"]: status["outlog_by_level"] = "" status["outlog_by_level"] += f"\n{HR}" status["outlog_by_level"] += salt.utils.stringutils.to_unicode( outlog_by_level ) status["logs"].extend( [(a[0], salt.utils.stringutils.to_unicode(a[1])) for a in logs] ) for log in logs_by_level: if log not in status["logs_by_level"]: status["logs_by_level"][log] = [] status["logs_by_level"][log].extend( [salt.utils.stringutils.to_unicode(a) for a in logs_by_level[log]] ) return _encode_status(status) @_salt_callback def buildout( directory=".", config="buildout.cfg", parts=None, runas=None, env=(), buildout_ver=None, test_release=False, distribute=None, new_st=None, offline=False, newest=False, python=sys.executable, debug=False, verbose=False, onlyif=None, unless=None, use_vt=False, loglevel=None, ): """ Run buildout in a directory. directory directory to execute in config buildout config to use parts specific buildout parts to run runas user used to run buildout as env environment variables to set when running buildout_ver force a specific buildout version (1 | 2) test_release buildout accept test release new_st Forcing use of setuptools >= 0.7 distribute use distribute over setuptools if possible offline does buildout run offline python python to use debug run buildout with -D debug flag onlyif Only execute cmd if statement on the host return 0 unless Do not execute cmd if statement on the host return 0 newest run buildout in newest mode verbose run buildout in verbose mode (-vvvvv) use_vt Use the new salt VT to stream output [experimental] CLI Example: .. code-block:: bash salt '*' buildout.buildout /srv/mybuildout """ LOG.info(f"Running buildout in {directory} ({config})") boot_ret = bootstrap( directory, config=config, buildout_ver=buildout_ver, test_release=test_release, offline=offline, new_st=new_st, env=env, runas=runas, distribute=distribute, python=python, use_vt=use_vt, loglevel=loglevel, ) buildout_ret = run_buildout( directory=directory, config=config, parts=parts, offline=offline, newest=newest, runas=runas, env=env, verbose=verbose, debug=debug, use_vt=use_vt, loglevel=loglevel, ) # signal the decorator or our return return _merge_statuses([boot_ret, buildout_ret]) def _check_onlyif_unless(onlyif, unless, directory, runas=None, env=()): ret = None status = BASE_STATUS.copy() if os.path.exists(directory): directory = os.path.abspath(directory) status["status"] = False retcode = __salt__["cmd.retcode"] if onlyif is not None: if not isinstance(onlyif, str): if not onlyif: _valid(status, "onlyif condition is false") elif isinstance(onlyif, str): if retcode(onlyif, cwd=directory, runas=runas, env=env) != 0: _valid(status, "onlyif condition is false") if unless is not None: if not isinstance(unless, str): if unless: _valid(status, "unless condition is true") elif isinstance(unless, str): if ( retcode( unless, cwd=directory, runas=runas, env=env, python_shell=False ) == 0 ): _valid(status, "unless condition is true") if status["status"]: ret = status return ret