# Copyright (C) 2012 Canonical Ltd.
# Copyright (C) 2012 Yahoo! Inc.
#
# Author: Scott Moser <scott.moser@canonical.com>
# Author: Joshua Harlow <harlowja@yahoo-inc.com>
#
# This file is part of cloud-init. See LICENSE file for license information.
import hashlib
import logging
import os
import time
from cloudinit import sources, url_helper, util
from cloudinit.net.cmdline import KlibcNetworkConfigSource
from cloudinit.sources import NetworkConfigSource
LOG = logging.getLogger(__name__)
MD_VERSION = "2012-03-01"
DS_FIELDS = [
# remote path, location in dictionary, binary data?, optional?
("meta-data/instance-id", "meta-data/instance-id", False, False),
("meta-data/local-hostname", "meta-data/local-hostname", False, False),
("meta-data/public-keys", "meta-data/public-keys", False, True),
("meta-data/vendor-data", "vendor-data", True, True),
("user-data", "user-data", True, True),
]
class DataSourceMAAS(sources.DataSource):
"""
DataSourceMAAS reads instance information from MAAS.
Given a config metadata_url, and oauth tokens, it expects to find
files under the root named:
instance-id
user-data
hostname
vendor-data
"""
dsname = "MAAS"
id_hash = None
_oauth_helper = None
def __init__(self, sys_cfg, distro, paths):
sources.DataSource.__init__(self, sys_cfg, distro, paths)
self.base_url = None
self.seed_dir = os.path.join(paths.seed_dir, "maas")
self.id_hash = get_id_from_ds_cfg(self.ds_cfg)
@property
def oauth_helper(self):
if not self._oauth_helper:
self._oauth_helper = get_oauth_helper(self.ds_cfg)
return self._oauth_helper
def __str__(self):
root = sources.DataSource.__str__(self)
return "%s [%s]" % (root, self.base_url)
def _get_data(self):
mcfg = self.ds_cfg
try:
self._set_data(self.seed_dir, read_maas_seed_dir(self.seed_dir))
return True
except MAASSeedDirNone:
pass
except MAASSeedDirMalformed as exc:
LOG.warning("%s was malformed: %s", self.seed_dir, exc)
raise
# If there is no metadata_url, then we're not configured
url = mcfg.get("metadata_url", None)
if not url:
return False
try:
# doing this here actually has a side affect of
# getting oauth time-fix in place. As no where else would
# retry by default, so even if we could fix the timestamp
# we would not.
if not self.wait_for_metadata_service(url):
return False
self._set_data(
url,
read_maas_seed_url(
url,
read_file_or_url=self.oauth_helper.readurl,
paths=self.paths,
retries=1,
),
)
return True
except Exception:
util.logexc(LOG, "Failed fetching metadata from url %s", url)
return False
def _set_data(self, url, data):
# takes a url for base_url and a tuple of userdata, metadata, vd.
self.base_url = url
ud, md, vd = data
self.userdata_raw = ud
self.metadata = md
self.vendordata_pure = vd
if vd:
try:
self.vendordata_raw = sources.convert_vendordata(vd)
except ValueError as e:
LOG.warning("Invalid content in vendor-data: %s", e)
self.vendordata_raw = None
def _get_subplatform(self):
"""Return the subplatform metadata source details."""
return "seed-dir (%s)" % self.base_url
def wait_for_metadata_service(self, url):
mcfg = self.ds_cfg
max_wait = 120
try:
max_wait = int(mcfg.get("max_wait", max_wait))
except Exception:
util.logexc(LOG, "Failed to get max wait. using %s", max_wait)
if max_wait == 0:
return False
timeout = 50
try:
if timeout in mcfg:
timeout = int(mcfg.get("timeout", timeout))
except Exception:
LOG.warning("Failed to get timeout, using %s", timeout)
starttime = time.time()
if url.endswith("/"):
url = url[:-1]
check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION)
urls = [check_url]
url, _response = self.oauth_helper.wait_for_url(
urls=urls, max_wait=max_wait, timeout=timeout
)
if url:
LOG.debug("Using metadata source: '%s'", url)
else:
LOG.critical(
"Giving up on md from %s after %i seconds",
urls,
int(time.time() - starttime),
)
return bool(url)
def check_instance_id(self, sys_cfg):
"""locally check if the current system is the same instance.
MAAS doesn't provide a real instance-id, and if it did, it is
still only available over the network. We need to check based
only on local resources. So compute a hash based on Oauth tokens."""
if self.id_hash is None:
return False
ncfg = util.get_cfg_by_path(sys_cfg, ("datasource", self.dsname), {})
return self.id_hash == get_id_from_ds_cfg(ncfg)
class DataSourceMAASLocal(DataSourceMAAS):
network_config_sources = (
NetworkConfigSource.CMD_LINE,
NetworkConfigSource.SYSTEM_CFG,
NetworkConfigSource.DS,
NetworkConfigSource.INITRAMFS,
)
def _get_data(self):
if not KlibcNetworkConfigSource().is_applicable():
# We booted from disk. Initramfs didn't bring up a network, so
# nothing to do. Wait until network timeframe to run _get_data()
LOG.debug("No initramfs applicable config")
return False
LOG.debug("Found initramfs applicable config")
return super()._get_data()
def get_oauth_helper(cfg):
"""Return an oauth helper instance for values in cfg.
@raises ValueError from OauthUrlHelper if some required fields have
true-ish values but others do not."""
keys = ("consumer_key", "consumer_secret", "token_key", "token_secret")
kwargs = dict([(r, cfg.get(r)) for r in keys])
return url_helper.OauthUrlHelper(**kwargs)
def get_id_from_ds_cfg(ds_cfg):
"""Given a config, generate a unique identifier for this node."""
fields = ("consumer_key", "token_key", "token_secret")
idstr = "\0".join([ds_cfg.get(f, "") for f in fields])
# store the encoding version as part of the hash in the event
# that it ever changed we can compute older versions.
return "v1:" + hashlib.sha256(idstr.encode("utf-8")).hexdigest()
def read_maas_seed_dir(seed_d):
if seed_d.startswith("file://"):
seed_d = seed_d[7:]
if not os.path.isdir(seed_d) or len(os.listdir(seed_d)) == 0:
raise MAASSeedDirNone("%s: not a directory")
# seed_dir looks in seed_dir, not seed_dir/VERSION
return read_maas_seed_url("file://%s" % seed_d, version=None)
def read_maas_seed_url(
seed_url,
read_file_or_url=None,
timeout=None,
version=MD_VERSION,
paths=None,
retries=None,
):
"""
Read the maas datasource at seed_url.
read_file_or_url is a method that should provide an interface
like util.read_file_or_url
Expected format of seed_url is are the following files:
* <seed_url>/<version>/meta-data/instance-id
* <seed_url>/<version>/meta-data/local-hostname
* <seed_url>/<version>/user-data
If version is None, then <version>/ will not be used.
"""
if read_file_or_url is None:
read_file_or_url = url_helper.read_file_or_url
if seed_url.endswith("/"):
seed_url = seed_url[:-1]
md = {}
for path, _dictname, binary, optional in DS_FIELDS:
if version is None:
url = "%s/%s" % (seed_url, path)
else:
url = "%s/%s/%s" % (seed_url, version, path)
try:
ssl_details = util.fetch_ssl_details(paths)
resp = read_file_or_url(
url, retries=retries, timeout=timeout, ssl_details=ssl_details
)
if resp.ok():
if binary:
md[path] = resp.contents
else:
md[path] = util.decode_binary(resp.contents)
else:
LOG.warning(
"Fetching from %s resulted in an invalid http code %s",
url,
resp.code,
)
except url_helper.UrlError as e:
if e.code == 404 and not optional:
raise MAASSeedDirMalformed(
"Missing required %s: %s" % (path, e)
) from e
elif e.code != 404:
raise e
return check_seed_contents(md, seed_url)
def check_seed_contents(content, seed):
"""Validate if dictionary content valid as a return for a datasource.
Either return a (userdata, metadata, vendordata) tuple or
Raise MAASSeedDirMalformed or MAASSeedDirNone
"""
ret = {}
missing = []
for spath, dpath, _binary, optional in DS_FIELDS:
if spath not in content:
if not optional:
missing.append(spath)
continue
if "/" in dpath:
top, _, p = dpath.partition("/")
if top not in ret:
ret[top] = {}
ret[top][p] = content[spath]
else:
ret[dpath] = content[spath]
if len(ret) == 0:
raise MAASSeedDirNone("%s: no data files found" % seed)
if missing:
raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing))
vd_data = None
if ret.get("vendor-data"):
err = object()
vd_data = util.load_yaml(
ret.get("vendor-data"), default=err, allowed=(object)
)
if vd_data is err:
raise MAASSeedDirMalformed("vendor-data was not loadable as yaml.")
return ret.get("user-data"), ret.get("meta-data"), vd_data
class MAASSeedDirNone(Exception):
pass
class MAASSeedDirMalformed(Exception):
pass
# Used to match classes to dependencies
datasources = [
(DataSourceMAAS, (sources.DEP_FILESYSTEM,)),
(DataSourceMAAS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
]
# Return a list of data sources that match this set of dependencies
def get_datasource_list(depends):
return sources.list_from_depends(depends, datasources)
if __name__ == "__main__":
def main():
"""
Call with single argument of directory or http or https url.
If url is given additional arguments are allowed, which will be
interpreted as consumer_key, token_key, token_secret, consumer_secret
"""
import argparse
import pprint
import sys
parser = argparse.ArgumentParser(description="Interact with MAAS DS")
parser.add_argument(
"--config",
metavar="file",
help="specify DS config file",
default=None,
)
parser.add_argument(
"--ckey",
metavar="key",
help="the consumer key to auth with",
default=None,
)
parser.add_argument(
"--tkey",
metavar="key",
help="the token key to auth with",
default=None,
)
parser.add_argument(
"--csec",
metavar="secret",
help="the consumer secret (likely '')",
default="",
)
parser.add_argument(
"--tsec",
metavar="secret",
help="the token secret to auth with",
default=None,
)
parser.add_argument(
"--apiver",
metavar="version",
help="the apiver to use ( can be used)",
default=MD_VERSION,
)
subcmds = parser.add_subparsers(title="subcommands", dest="subcmd")
for (name, help) in (
("crawl", "crawl the datasource"),
("get", "do a single GET of provided url"),
("check-seed", "read and verify seed at url"),
):
p = subcmds.add_parser(name, help=help)
p.add_argument(
"url", help="the datasource url", nargs="?", default=None
)
args = parser.parse_args()
creds = {
"consumer_key": args.ckey,
"token_key": args.tkey,
"token_secret": args.tsec,
"consumer_secret": args.csec,
}
if args.config is None:
for fname in ("91_kernel_cmdline_url", "90_dpkg_maas"):
fpath = "/etc/cloud/cloud.cfg.d/" + fname + ".cfg"
if os.path.exists(fpath) and os.access(fpath, os.R_OK):
sys.stderr.write("Used config in %s.\n" % fpath)
args.config = fpath
if args.config:
cfg = util.read_conf(args.config)
if "datasource" in cfg:
cfg = cfg["datasource"]["MAAS"]
for key in creds.keys():
if key in cfg and creds[key] is None:
creds[key] = cfg[key]
if args.url is None and "metadata_url" in cfg:
args.url = cfg["metadata_url"]
if args.url is None:
sys.stderr.write("Must provide a url or a config with url.\n")
sys.exit(1)
oauth_helper = get_oauth_helper(creds)
def geturl(url):
# the retry is to ensure that oauth timestamp gets fixed
return oauth_helper.readurl(url, retries=1).contents
def printurl(url):
print("== %s ==\n%s\n" % (url, geturl(url).decode()))
def crawl(url):
if url.endswith("/"):
for line in geturl(url).decode().splitlines():
if line.endswith("/"):
crawl("%s%s" % (url, line))
elif line == "meta-data":
# meta-data is a dir, it *should* end in a /
crawl("%s%s" % (url, "meta-data/"))
else:
printurl("%s%s" % (url, line))
else:
printurl(url)
if args.subcmd == "check-seed":
sys.stderr.write("Checking seed at %s\n" % args.url)
readurl = oauth_helper.readurl
if args.url[0] == "/" or args.url.startswith("file://"):
(userdata, metadata, vd) = read_maas_seed_dir(args.url)
else:
(userdata, metadata, vd) = read_maas_seed_url(
args.url,
version=args.apiver,
read_file_or_url=readurl,
retries=2,
)
print("=== user-data ===")
print("N/A" if userdata is None else userdata.decode())
print("=== meta-data ===")
pprint.pprint(metadata)
print("=== vendor-data ===")
pprint.pprint("N/A" if vd is None else vd)
elif args.subcmd == "get":
printurl(args.url)
elif args.subcmd == "crawl":
if not args.url.endswith("/"):
args.url = "%s/" % args.url
crawl(args.url)
main()