| Dir : /proc/self/root/opt/saltstack/salt/lib/python3.10/site-packages/salt/utils/ |
| Server: Linux ngx353.inmotionhosting.com 4.18.0-553.22.1.lve.1.el8.x86_64 #1 SMP Tue Oct 8 15:52:54 UTC 2024 x86_64 IP: 209.182.202.254 |
| Dir : //proc/self/root/opt/saltstack/salt/lib/python3.10/site-packages/salt/utils/http.py |
"""
Utils for making various web calls. Primarily designed for REST, SOAP, webhooks
and the like, but also useful for basic HTTP testing.
.. versionadded:: 2015.5.0
"""
import email.message
import gzip
import http.client
import http.cookiejar
import io
import logging
import os
import pprint
import re
import socket
import ssl
import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
import zlib
import tornado.httpclient
import tornado.httputil
import tornado.simple_httpclient
from tornado.httpclient import HTTPClient
import salt.config
import salt.loader
import salt.syspaths
import salt.utils.args
import salt.utils.data
import salt.utils.files
import salt.utils.json
import salt.utils.msgpack
import salt.utils.network
import salt.utils.platform
import salt.utils.stringutils
import salt.utils.url
import salt.utils.xmlutil as xml
import salt.utils.yaml
import salt.version
from salt.template import compile_template
from salt.utils.decorators.jinja import jinja_filter
try:
from ssl import CertificateError, match_hostname
HAS_MATCHHOSTNAME = True
except ImportError:
# pylint: disable=no-name-in-module
try:
from backports.ssl_match_hostname import CertificateError, match_hostname
HAS_MATCHHOSTNAME = True
except ImportError:
try:
from salt.ext.ssl_match_hostname import CertificateError, match_hostname
HAS_MATCHHOSTNAME = True
except ImportError:
HAS_MATCHHOSTNAME = False
# pylint: enable=no-name-in-module
try:
import requests
HAS_REQUESTS = True
except ImportError:
HAS_REQUESTS = False
try:
import certifi
HAS_CERTIFI = True
except ImportError:
HAS_CERTIFI = False
log = logging.getLogger(__name__)
USERAGENT = f"Salt/{salt.version.__version__}"
def __decompressContent(coding, pgctnt):
"""
Decompress returned HTTP content depending on the specified encoding.
Currently supports identity/none, deflate, and gzip, which should
cover 99%+ of the content on the internet.
"""
if not pgctnt:
return pgctnt
log.trace(
"Decompressing %s byte content with compression type: %s", len(pgctnt), coding
)
if coding == "deflate":
pgctnt = zlib.decompress(pgctnt, -zlib.MAX_WBITS)
elif coding == "gzip":
buf = io.BytesIO(pgctnt)
f = gzip.GzipFile(fileobj=buf)
pgctnt = f.read()
elif coding == "sdch":
raise ValueError("SDCH compression is not currently supported")
elif coding == "br":
raise ValueError("Brotli compression is not currently supported")
elif coding == "compress":
raise ValueError("LZW compression is not currently supported")
log.trace("Content size after decompression: %s", len(pgctnt))
return pgctnt
def _decode_result_text(result_text, backend, decode_body=None, result=None):
"""
Decode only the result_text
"""
if backend == "requests":
if not isinstance(result_text, str) and decode_body:
result_text = result_text.decode(result.encoding or "utf-8")
else:
if isinstance(result_text, bytes) and decode_body:
result_text = result_text.decode("utf-8")
return result_text
def _decode_result(result_text, result_headers, backend, decode_body=None, result=None):
"""
Decode the result_text and headers.
"""
if "Content-Type" in result_headers:
msg = email.message.EmailMessage()
msg.add_header("Content-Type", result_headers["Content-Type"])
if msg.get_content_type().startswith("text/"):
content_charset = msg.get_content_charset()
if content_charset and not isinstance(result_text, str):
result_text = result_text.decode(content_charset)
result_text = _decode_result_text(
result_text, backend, decode_body=decode_body, result=result
)
return result_text, result_headers
@jinja_filter("http_query")
def query(
url,
method="GET",
params=None,
data=None,
data_file=None,
header_dict=None,
header_list=None,
header_file=None,
username=None,
password=None,
auth=None,
decode=False,
decode_type="auto",
status=False,
headers=False,
text=False,
cookies=None,
cookie_jar=None,
cookie_format="lwp",
persist_session=False,
session_cookie_jar=None,
data_render=False,
data_renderer=None,
header_render=False,
header_renderer=None,
template_dict=None,
test=False,
test_url=None,
node="minion",
port=80,
opts=None,
backend=None,
ca_bundle=None,
verify_ssl=None,
cert=None,
text_out=None,
headers_out=None,
decode_out=None,
stream=False,
streaming_callback=None,
header_callback=None,
handle=False,
agent=USERAGENT,
hide_fields=None,
raise_error=True,
formdata=False,
formdata_fieldname=None,
formdata_filename=None,
decode_body=True,
**kwargs,
):
"""
Query a resource, and decode the return data
"""
ret = {}
if opts is None:
if node == "master":
opts = salt.config.master_config(
os.path.join(salt.syspaths.CONFIG_DIR, "master")
)
elif node == "minion":
opts = salt.config.minion_config(
os.path.join(salt.syspaths.CONFIG_DIR, "minion")
)
else:
opts = {}
if not backend:
backend = opts.get("backend", "tornado")
proxy_host = opts.get("proxy_host", None)
if proxy_host:
proxy_host = salt.utils.stringutils.to_str(proxy_host)
proxy_port = opts.get("proxy_port", None)
proxy_username = opts.get("proxy_username", None)
if proxy_username:
proxy_username = salt.utils.stringutils.to_str(proxy_username)
proxy_password = opts.get("proxy_password", None)
if proxy_password:
proxy_password = salt.utils.stringutils.to_str(proxy_password)
no_proxy = opts.get("no_proxy", [])
if urllib.parse.urlparse(url).hostname in no_proxy:
proxy_host = None
proxy_port = None
proxy_username = None
proxy_password = None
http_proxy_url = None
if proxy_host and proxy_port:
if backend != "requests":
log.debug("Switching to request backend due to the use of proxies.")
backend = "requests"
if proxy_username and proxy_password:
http_proxy_url = (
f"http://{proxy_username}:{proxy_password}@{proxy_host}:{proxy_port}"
)
else:
http_proxy_url = f"http://{proxy_host}:{proxy_port}"
match = re.match(
r"https?://((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)($|/)",
url,
)
if not match:
salt.utils.network.refresh_dns()
if backend == "requests":
if HAS_REQUESTS is False:
ret["error"] = (
"http.query has been set to use requests, but the "
"requests library does not seem to be installed"
)
log.error(ret["error"])
return ret
else:
requests_log = logging.getLogger("requests")
requests_log.setLevel(logging.WARNING)
# Some libraries don't support separation of url and GET parameters
# Don't need a try/except block, since Salt depends on tornado
url_full = tornado.httputil.url_concat(url, params) if params else url
if ca_bundle is None:
ca_bundle = get_ca_bundle(opts)
if verify_ssl is None:
verify_ssl = opts.get("verify_ssl", True)
if cert is None:
cert = opts.get("cert", None)
if data_file is not None:
data = _render(data_file, data_render, data_renderer, template_dict, opts)
# Make sure no secret fields show up in logs
log_url = sanitize_url(url_full, hide_fields)
log.debug("Requesting URL %s using %s method", log_url, method)
log.debug("Using backend: %s", backend)
if method == "POST" and log.isEnabledFor(logging.TRACE):
# Make sure no secret fields show up in logs
if isinstance(data, dict):
log_data = data.copy()
if isinstance(hide_fields, list):
for item in data:
for field in hide_fields:
if item == field:
log_data[item] = "XXXXXXXXXX"
log.trace("Request POST Data: %s", pprint.pformat(log_data))
else:
log.trace("Request POST Data: %s", pprint.pformat(data))
if header_file is not None:
header_tpl = _render(
header_file, header_render, header_renderer, template_dict, opts
)
if isinstance(header_tpl, dict):
header_dict = header_tpl
else:
header_list = header_tpl.splitlines()
if header_dict is None:
header_dict = {}
if header_list is None:
header_list = []
if cookie_jar is None:
cookie_jar = os.path.join(
opts.get("cachedir", salt.syspaths.CACHE_DIR), "cookies.txt"
)
if session_cookie_jar is None:
session_cookie_jar = os.path.join(
opts.get("cachedir", salt.syspaths.CACHE_DIR), "cookies.session.p"
)
if persist_session is True and salt.utils.msgpack.HAS_MSGPACK:
# TODO: This is hackish; it will overwrite the session cookie jar with
# all cookies from this one connection, rather than behaving like a
# proper cookie jar. Unfortunately, since session cookies do not
# contain expirations, they can't be stored in a proper cookie jar.
if os.path.isfile(session_cookie_jar):
with salt.utils.files.fopen(session_cookie_jar, "rb") as fh_:
session_cookies = salt.utils.msgpack.load(fh_)
if isinstance(session_cookies, dict):
header_dict.update(session_cookies)
else:
with salt.utils.files.fopen(session_cookie_jar, "wb") as fh_:
salt.utils.msgpack.dump("", fh_)
for header in header_list:
comps = header.split(":")
if len(comps) < 2:
continue
header_dict[comps[0].strip()] = comps[1].strip()
if not auth:
if username and password:
auth = (username, password)
if agent == USERAGENT:
agent = f"{agent} http.query()"
header_dict["User-agent"] = agent
if (
proxy_host
and proxy_port
and method == "POST"
and "Content-Type" not in header_dict
):
log.debug(
"Content-Type not provided for POST request, assuming application/x-www-form-urlencoded"
)
header_dict["Content-Type"] = "application/x-www-form-urlencoded"
if "Content-Length" not in header_dict:
header_dict["Content-Length"] = f"{len(data)}"
if backend == "requests":
sess = requests.Session()
sess.auth = auth
sess.headers.update(header_dict)
log.trace("Request Headers: %s", sess.headers)
sess_cookies = sess.cookies
sess.verify = verify_ssl
if http_proxy_url is not None:
sess.proxies = {
"http": http_proxy_url,
"https": http_proxy_url,
}
elif backend == "urllib2":
sess_cookies = None
else:
# Tornado
sess_cookies = None
if cookies is not None:
if cookie_format == "mozilla":
sess_cookies = http.cookiejar.MozillaCookieJar(cookie_jar)
else:
sess_cookies = http.cookiejar.LWPCookieJar(cookie_jar)
if not os.path.isfile(cookie_jar):
sess_cookies.save()
sess_cookies.load()
if test is True:
if test_url is None:
return {}
else:
url = test_url
ret["test"] = True
if backend == "requests":
req_kwargs = {}
if stream is True:
if requests.__version__[0] == "0":
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
req_kwargs["prefetch"] = False
else:
req_kwargs["stream"] = True
# Client-side cert handling
if cert is not None:
if isinstance(cert, str):
if os.path.exists(cert):
req_kwargs["cert"] = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs["cert"] = cert
else:
log.error(
"The client-side certificate path that was passed is not valid: %s",
cert,
)
if formdata:
if not formdata_fieldname:
ret["error"] = "formdata_fieldname is required when formdata=True"
log.error(ret["error"])
return ret
result = sess.request(
method,
url,
params=params,
files={formdata_fieldname: (formdata_filename, io.StringIO(data))},
**req_kwargs,
)
else:
result = sess.request(method, url, params=params, data=data, **req_kwargs)
result.raise_for_status()
if stream is True:
# fake a HTTP response header
header_callback(f"HTTP/1.0 {result.status_code} MESSAGE")
# fake streaming the content
streaming_callback(result.content)
return {
"handle": result,
}
if handle is True:
return {
"handle": result,
"body": result.content,
}
log.debug(
"Final URL location of Response: %s", sanitize_url(result.url, hide_fields)
)
result_status_code = result.status_code
result_headers = result.headers
result_text = result.content
result_cookies = result.cookies
result_text = _decode_result_text(
result_text, backend, decode_body=decode_body, result=result
)
ret["body"] = result_text
elif backend == "urllib2":
request = urllib.request.Request(url_full, data)
handlers = [
urllib.request.HTTPHandler,
urllib.request.HTTPCookieProcessor(sess_cookies),
]
if url.startswith("https"):
hostname = request.get_host()
handlers[0] = urllib.request.HTTPSHandler(1)
if not HAS_MATCHHOSTNAME:
log.warning(
"match_hostname() not available, SSL hostname checking "
"not available. THIS CONNECTION MAY NOT BE SECURE!"
)
elif verify_ssl is False:
log.warning(
"SSL certificate verification has been explicitly "
"disabled. THIS CONNECTION MAY NOT BE SECURE!"
)
else:
if ":" in hostname:
hostname, port = hostname.split(":")
else:
port = 443
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((hostname, int(port)))
sockwrap = ssl.wrap_socket(
sock, ca_certs=ca_bundle, cert_reqs=ssl.CERT_REQUIRED
)
try:
match_hostname(sockwrap.getpeercert(), hostname)
except CertificateError as exc:
ret["error"] = (
"The certificate was invalid. Error returned was: {}".format(
pprint.pformat(exc)
)
)
return ret
# Client-side cert handling
if cert is not None:
cert_chain = None
if isinstance(cert, str):
if os.path.exists(cert):
cert_chain = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
cert_chain = cert
else:
log.error(
"The client-side certificate path that was "
"passed is not valid: %s",
cert,
)
return
if hasattr(ssl, "SSLContext"):
# Python >= 2.7.9
context = ssl.SSLContext.load_cert_chain(*cert_chain)
handlers.append(
urllib.request.HTTPSHandler(context=context)
) # pylint: disable=E1123
else:
# Python < 2.7.9
cert_kwargs = {
"host": request.get_host(),
"port": port,
"cert_file": cert_chain[0],
}
if len(cert_chain) > 1:
cert_kwargs["key_file"] = cert_chain[1]
handlers[0] = http.client.HTTPSConnection(**cert_kwargs)
opener = urllib.request.build_opener(*handlers)
for header in header_dict:
request.add_header(header, header_dict[header])
request.get_method = lambda: method
try:
result = opener.open(request)
except urllib.error.URLError as exc:
return {"Error": str(exc)}
if stream is True or handle is True:
return {
"handle": result,
"body": result.content,
}
result_status_code = result.code
result_headers = dict(result.info())
result_text = result.read()
result_text, result_headers = _decode_result(
result_text, result_headers, backend, decode_body=decode_body, result=result
)
ret["body"] = result_text
else:
# Tornado
req_kwargs = {}
# Client-side cert handling
if cert is not None:
if isinstance(cert, str):
if os.path.exists(cert):
req_kwargs["client_cert"] = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs["client_cert"] = cert[0]
req_kwargs["client_key"] = cert[1]
else:
log.error(
"The client-side certificate path that was passed is not valid: %s",
cert,
)
if isinstance(data, dict):
data = urllib.parse.urlencode(data)
if verify_ssl:
req_kwargs["ca_certs"] = ca_bundle
max_body = opts.get(
"http_max_body", salt.config.DEFAULT_MINION_OPTS["http_max_body"]
)
connect_timeout = opts.get(
"http_connect_timeout",
salt.config.DEFAULT_MINION_OPTS["http_connect_timeout"],
)
timeout = opts.get(
"http_request_timeout",
salt.config.DEFAULT_MINION_OPTS["http_request_timeout"],
)
tornado.httpclient.AsyncHTTPClient.configure(None)
client_argspec = salt.utils.args.get_function_argspec(
tornado.simple_httpclient.SimpleAsyncHTTPClient.initialize
)
supports_max_body_size = "max_body_size" in client_argspec.args
req_kwargs.update(
{
"method": method,
"headers": header_dict,
"auth_username": username,
"auth_password": password,
"body": data,
"validate_cert": verify_ssl,
"allow_nonstandard_methods": True,
"streaming_callback": streaming_callback,
"header_callback": header_callback,
"connect_timeout": connect_timeout,
"request_timeout": timeout,
"raise_error": raise_error,
"decompress_response": False,
}
)
# Unicode types will cause a TypeError when Tornado's curl HTTPClient
# invokes setopt. Therefore, make sure all arguments we pass which
# contain strings are str types.
req_kwargs = salt.utils.data.decode(req_kwargs, to_str=True)
try:
download_client = (
HTTPClient(max_body_size=max_body)
if supports_max_body_size
else HTTPClient()
)
result = download_client.fetch(url_full, **req_kwargs)
except tornado.httpclient.HTTPError as exc:
ret["status"] = exc.code
ret["error"] = str(exc)
ret["body"], _ = _decode_result(
exc.response.body,
exc.response.headers,
backend,
decode_body=decode_body,
)
return ret
except (socket.herror, OSError, socket.timeout, socket.gaierror) as exc:
if status is True:
ret["status"] = 0
ret["error"] = str(exc)
log.debug("Cannot perform 'http.query': %s - %s", url_full, ret["error"])
return ret
if stream is True or handle is True:
return {
"handle": result,
"body": result.body,
}
result_status_code = result.code
result_headers = result.headers
result_text = result.body
result_text, result_headers = _decode_result(
result_text, result_headers, backend, decode_body=decode_body, result=result
)
ret["body"] = result_text
if "Set-Cookie" in result_headers and cookies is not None:
result_cookies = parse_cookie_header(result_headers["Set-Cookie"])
for item in result_cookies:
sess_cookies.set_cookie(item)
else:
result_cookies = None
if isinstance(result_headers, list):
result_headers_dict = {}
for header in result_headers:
comps = header.split(":")
result_headers_dict[comps[0].strip()] = ":".join(comps[1:]).strip()
result_headers = result_headers_dict
log.debug("Response Status Code: %s", result_status_code)
log.trace("Response Headers: %s", result_headers)
log.trace("Response Cookies: %s", sess_cookies)
# log.trace("Content: %s", result_text)
coding = result_headers.get("Content-Encoding", "identity")
# Requests will always decompress the content, and working around that is annoying.
if backend != "requests":
result_text = __decompressContent(coding, result_text)
try:
log.trace("Response Text: %s", result_text)
except UnicodeEncodeError as exc:
log.trace(
"Cannot Trace Log Response Text: %s. This may be due to "
"incompatibilities between requests and logging.",
exc,
)
if text_out is not None:
with salt.utils.files.fopen(text_out, "w") as tof:
tof.write(result_text)
if headers_out is not None and os.path.exists(headers_out):
with salt.utils.files.fopen(headers_out, "w") as hof:
hof.write(result_headers)
if cookies is not None:
sess_cookies.save()
if persist_session is True and salt.utils.msgpack.HAS_MSGPACK:
# TODO: See persist_session above
if "set-cookie" in result_headers:
with salt.utils.files.fopen(session_cookie_jar, "wb") as fh_:
session_cookies = result_headers.get("set-cookie", None)
if session_cookies is not None:
salt.utils.msgpack.dump({"Cookie": session_cookies}, fh_)
else:
salt.utils.msgpack.dump("", fh_)
if status is True:
ret["status"] = result_status_code
if headers is True:
ret["headers"] = result_headers
if decode is True:
if decode_type == "auto":
content_type = result_headers.get("content-type", "application/json")
if "xml" in content_type:
decode_type = "xml"
elif "json" in content_type:
decode_type = "json"
elif "yaml" in content_type:
decode_type = "yaml"
else:
decode_type = "plain"
valid_decodes = ("json", "xml", "yaml", "plain")
if decode_type not in valid_decodes:
ret["error"] = (
"Invalid decode_type specified. Valid decode types are: {}".format(
pprint.pformat(valid_decodes)
)
)
log.error(ret["error"])
return ret
if decode_type == "json":
ret["dict"] = salt.utils.json.loads(result_text)
elif decode_type == "xml":
ret["dict"] = []
items = ET.fromstring(result_text)
for item in items:
ret["dict"].append(xml.to_dict(item))
elif decode_type == "yaml":
ret["dict"] = salt.utils.data.decode(salt.utils.yaml.safe_load(result_text))
else:
text = True
if decode_out:
with salt.utils.files.fopen(decode_out, "w") as dof:
dof.write(result_text)
if text is True:
ret["text"] = result_text
return ret
def get_ca_bundle(opts=None):
"""
Return the location of the ca bundle file. See the following article:
http://tinyurl.com/k7rx42a
"""
if hasattr(get_ca_bundle, "__return_value__"):
return get_ca_bundle.__return_value__
if opts is None:
opts = {}
opts_bundle = opts.get("ca_bundle", None)
if opts_bundle is not None and os.path.exists(opts_bundle):
return opts_bundle
file_roots = opts.get("file_roots", {"base": [salt.syspaths.SRV_ROOT_DIR]})
# Please do not change the order without good reason
# Check Salt first
for salt_root in file_roots.get("base", []):
for path in ("cacert.pem", "ca-bundle.crt"):
cert_path = os.path.join(salt_root, path)
if os.path.exists(cert_path):
return cert_path
locations = (
# Debian has paths that often exist on other distros
"/etc/ssl/certs/ca-certificates.crt",
# RedHat is also very common
"/etc/pki/tls/certs/ca-bundle.crt",
"/etc/pki/tls/certs/ca-bundle.trust.crt",
# RedHat's link for Debian compatibility
"/etc/ssl/certs/ca-bundle.crt",
# SUSE has an unusual path
"/var/lib/ca-certificates/ca-bundle.pem",
# OpenBSD has an unusual path
"/etc/ssl/cert.pem",
)
for path in locations:
if os.path.exists(path):
return path
if salt.utils.platform.is_windows() and HAS_CERTIFI:
return certifi.where()
return None
def update_ca_bundle(
target=None,
source=None,
opts=None,
merge_files=None,
):
"""
Attempt to update the CA bundle file from a URL
If not specified, the local location on disk (``target``) will be
auto-detected, if possible. If it is not found, then a new location on disk
will be created and updated.
The default ``source`` is:
http://curl.haxx.se/ca/cacert.pem
This is based on the information at:
http://curl.haxx.se/docs/caextract.html
A string or list of strings representing files to be appended to the end of
the CA bundle file may also be passed through as ``merge_files``.
"""
if opts is None:
opts = {}
if target is None:
target = get_ca_bundle(opts)
if target is None:
log.error("Unable to detect location to write CA bundle to")
return
if source is None:
source = opts.get("ca_bundle_url", "http://curl.haxx.se/ca/cacert.pem")
log.debug("Attempting to download %s to %s", source, target)
query(source, text=True, decode=False, headers=False, status=False, text_out=target)
if merge_files is not None:
if isinstance(merge_files, str):
merge_files = [merge_files]
if not isinstance(merge_files, list):
log.error(
"A value was passed as merge_files which was not either "
"a string or a list"
)
return
merge_content = ""
for cert_file in merge_files:
if os.path.exists(cert_file):
log.debug("Queueing up %s to be appended to %s", cert_file, target)
try:
with salt.utils.files.fopen(cert_file, "r") as fcf:
merge_content = "\n".join((merge_content, fcf.read()))
except OSError as exc:
log.error(
"Reading from %s caused the following error: %s", cert_file, exc
)
if merge_content:
log.debug("Appending merge_files to %s", target)
try:
with salt.utils.files.fopen(target, "a") as tfp:
tfp.write("\n")
tfp.write(merge_content)
except OSError as exc:
log.error("Writing to %s caused the following error: %s", target, exc)
def _render(template, render, renderer, template_dict, opts):
"""
Render a template
"""
if render:
if template_dict is None:
template_dict = {}
if not renderer:
renderer = opts.get("renderer", "jinja|yaml")
rend = salt.loader.render(opts, {})
blacklist = opts.get("renderer_blacklist")
whitelist = opts.get("renderer_whitelist")
ret = compile_template(
template, rend, renderer, blacklist, whitelist, **template_dict
)
if salt.utils.stringio.is_readable(ret):
ret = ret.read()
if str(ret).startswith("#!") and not str(ret).startswith("#!/"):
ret = str(ret).split("\n", 1)[1]
return ret
with salt.utils.files.fopen(template, "r") as fh_:
return fh_.read()
def parse_cookie_header(header):
"""
Parse the "Set-cookie" header, and return a list of cookies.
This function is here because Tornado's HTTPClient doesn't handle cookies.
"""
attribs = (
"expires",
"path",
"domain",
"version",
"httponly",
"secure",
"comment",
"max-age",
"samesite",
)
# Split into cookie(s); handles headers with multiple cookies defined
morsels = []
for item in header.split(";"):
item = item.strip()
if "," in item and "expires" not in item:
for part in item.split(","):
morsels.append(part)
else:
morsels.append(item)
# Break down morsels into actual cookies
cookies = []
cookie = {}
value_set = False
for morsel in morsels:
parts = morsel.split("=")
parts[0] = parts[0].lower()
if parts[0] in attribs:
if parts[0] in cookie:
cookies.append(cookie)
cookie = {}
if len(parts) > 1:
cookie[parts[0]] = "=".join(parts[1:])
else:
cookie[parts[0]] = True
else:
if value_set is True:
# This is a new cookie; save the old one and clear for this one
cookies.append(cookie)
cookie = {}
value_set = False
cookie[parts[0]] = "=".join(parts[1:])
value_set = True
if cookie:
# Set the last cookie that was processed
cookies.append(cookie)
# These arguments are required by cookielib.Cookie()
reqd = (
"version",
"port",
"port_specified",
"domain",
"domain_specified",
"domain_initial_dot",
"path",
"path_specified",
"secure",
"expires",
"discard",
"comment",
"comment_url",
"rest",
)
ret = []
for cookie in cookies:
name = None
value = None
for item in list(cookie):
if item in attribs:
continue
name = item
value = cookie.pop(item)
# cookielib.Cookie() requires an epoch
if "expires" in cookie:
cookie["expires"] = http.cookiejar.http2time(cookie["expires"])
# Fill in missing required fields
for req in reqd:
if req not in cookie:
cookie[req] = ""
if cookie["version"] == "":
cookie["version"] = 0
if cookie["rest"] == "":
cookie["rest"] = {}
if cookie["expires"] == "":
cookie["expires"] = 0
# Remove attribs that don't apply to Cookie objects
cookie.pop("httponly", None)
cookie.pop("samesite", None)
ret.append(http.cookiejar.Cookie(name=name, value=value, **cookie))
return ret
def sanitize_url(url, hide_fields):
"""
Make sure no secret fields show up in logs
"""
if isinstance(hide_fields, list):
url_comps = urllib.parse.splitquery(url)
log_url = url_comps[0]
if len(url_comps) > 1:
log_url += "?"
for pair in url_comps[1:]:
url_tmp = None
for field in hide_fields:
comps_list = pair.split("&")
if url_tmp:
url_tmp = url_tmp.split("&")
url_tmp = _sanitize_url_components(url_tmp, field)
else:
url_tmp = _sanitize_url_components(comps_list, field)
log_url += url_tmp
return log_url.rstrip("&")
else:
return str(url)
def _sanitize_url_components(comp_list, field):
"""
Recursive function to sanitize each component of the url.
"""
if not comp_list:
return ""
elif comp_list[0].startswith(f"{field}="):
ret = f"{field}=XXXXXXXXXX&"
comp_list.remove(comp_list[0])
return ret + _sanitize_url_components(comp_list, field)
else:
ret = f"{comp_list[0]}&"
comp_list.remove(comp_list[0])
return ret + _sanitize_url_components(comp_list, field)
def session(user=None, password=None, verify_ssl=True, ca_bundle=None, headers=None):
"""
create a requests session
"""
session = requests.session()
if user and password:
session.auth = (user, password)
if ca_bundle and not verify_ssl:
log.error("You cannot use both ca_bundle and verify_ssl False together")
return False
if ca_bundle:
opts = {"ca_bundle": ca_bundle}
session.verify = get_ca_bundle(opts)
if not verify_ssl:
session.verify = False
if headers:
session.headers.update(headers)
return session