500 lines
19 KiB
Python
500 lines
19 KiB
Python
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
|
# coding=utf-8
|
|
|
|
# Copyright 2013 Hewlett-Packard Development Company, L.P.
|
|
# Copyright 2013 International Business Machines Corporation
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
# Copyright (c) 2020 Wind River Systems, Inc.
|
|
#
|
|
|
|
|
|
""" Perform activity related to FPGA devices on a single host.
|
|
|
|
A single instance of :py:class:`sysinv.agent.manager.FpgaAgentManager` is
|
|
created within the *sysinv-fpga-agent* process, and is responsible for
|
|
performing all actions for this host related to FPGA devices.
|
|
|
|
On start, collect and post FPGA inventory to conductor.
|
|
|
|
Commands (from conductors) are received via RPC calls.
|
|
|
|
"""
|
|
|
|
from __future__ import print_function
|
|
import errno
|
|
from eventlet.green import subprocess
|
|
from glob import glob
|
|
|
|
import os
|
|
import shlex
|
|
import shutil
|
|
import time
|
|
import tsconfig.tsconfig as tsc
|
|
import urllib
|
|
|
|
from oslo_config import cfg
|
|
from oslo_log import log
|
|
from oslo_utils import uuidutils
|
|
|
|
from sysinv.common import device as dconstants
|
|
from sysinv.common import exception
|
|
from sysinv.common import service
|
|
from sysinv.conductor import rpcapi as conductor_rpcapi
|
|
from sysinv.objects import base as objects_base
|
|
from sysinv.openstack.common import context as ctx
|
|
|
|
|
|
MANAGER_TOPIC = 'sysinv.fpga_agent_manager'
|
|
|
|
LOG = log.getLogger(__name__)
|
|
|
|
agent_opts = [
|
|
cfg.StrOpt('api_url',
|
|
default=None,
|
|
help=('Url of SysInv API service. If not set SysInv can '
|
|
'get current value from Keystone service catalog.')),
|
|
cfg.IntOpt('audit_interval',
|
|
default=60,
|
|
help='Maximum time since the last check-in of a agent'),
|
|
]
|
|
|
|
CONF = cfg.CONF
|
|
CONF.register_opts(agent_opts, 'fpga_agent')
|
|
|
|
# Currently we only support the following FPGA. In the future we may need to
|
|
# expand this to a list of devices, each with their own special set of
|
|
# device-specific information.
|
|
FPGA_VENDOR = "8086"
|
|
FPGA_DEVICE = "0b30"
|
|
|
|
# TODO: Make this specified in the config file.
|
|
# This is the docker image containing the OPAE tools to access the FPGA device.
|
|
OPAE_IMG = "registry.local:9001/docker.io/starlingx/n3000-opae:stx.4.0-v1.0.0"
|
|
|
|
# This is the location where we cache the device image file while
|
|
# writing it to the hardware.
|
|
DEVICE_IMAGE_CACHE_DIR = "/usr/local/share/applications/sysinv"
|
|
|
|
SYSFS_DEVICE_PATH = "/sys/bus/pci/devices/"
|
|
FME_PATH = "/fpga/intel-fpga-dev.*/intel-fpga-fme.*/"
|
|
SPI_PATH = "spi-altera.*.auto/spi_master/spi*/spi*.*/"
|
|
|
|
# These are relative to FME_PATH
|
|
BITSTREAM_ID_PATH = "bitstream_id"
|
|
|
|
# These are relative to SPI_PATH
|
|
ROOT_HASH_PATH = "ifpga_sec_mgr/ifpga_sec*/security/sr_root_hash"
|
|
CANCELLED_CSKS_PATH = "ifpga_sec_mgr/ifpga_sec*/security/sr_canceled_csks"
|
|
IMAGE_LOAD_PATH = "fpga_flash_ctrl/fpga_image_load"
|
|
BMC_FW_VER_PATH = "bmcfw_flash_ctrl/bmcfw_version"
|
|
BMC_BUILD_VER_PATH = "max10_version"
|
|
|
|
|
|
def ensure_device_image_cache_exists():
|
|
# Make sure the image cache directory exists, create it if needed.
|
|
try:
|
|
os.mkdir(DEVICE_IMAGE_CACHE_DIR, 0o755)
|
|
except OSError as exc:
|
|
if exc.errno != errno.EEXIST:
|
|
msg = ("Unable to create device image cache directory %s!"
|
|
% DEVICE_IMAGE_CACHE_DIR)
|
|
LOG.exception(msg)
|
|
raise exception.SysinvException(msg)
|
|
|
|
|
|
def fetch_device_image(filename):
|
|
# Pull the image from the controller.
|
|
url = "http://controller:8080/device_images/" + filename
|
|
local_path = DEVICE_IMAGE_CACHE_DIR + "/" + filename
|
|
try:
|
|
imagefile, headers = urllib.urlretrieve(url, local_path)
|
|
except IOError:
|
|
msg = ("Unable to retrieve device image from %s!" % url)
|
|
LOG.exception(msg)
|
|
raise exception.SysinvException(msg)
|
|
return local_path
|
|
|
|
|
|
def fetch_device_image_local(filename):
|
|
# This is a hack since we only support AIO for now. Just copy the device
|
|
# image file into the well-known device image cache directory.
|
|
local_path = DEVICE_IMAGE_CACHE_DIR + "/" + filename
|
|
image_file_path = os.path.join(dconstants.DEVICE_IMAGE_PATH, filename)
|
|
try:
|
|
shutil.copyfile(image_file_path, local_path)
|
|
except (shutil.Error, IOError):
|
|
msg = ("Unable to retrieve device image from %s!" % image_file_path)
|
|
LOG.exception(msg)
|
|
raise exception.SysinvException(msg)
|
|
return local_path
|
|
|
|
|
|
def write_device_image_n3000(filename, pci_addr):
|
|
# Write the firmware image to the FPGA at the specified PCI address.
|
|
# We're assuming that the image update tools will catch the scenario
|
|
# where the image is not compatible with the device.
|
|
try:
|
|
# Build up the command to perform the firmware update.
|
|
# Note the hack to work around OPAE tool locale issues
|
|
cmd = ("docker run -t --privileged -e LC_ALL=en_US.UTF-8 "
|
|
"-e LANG=en_US.UTF-8 -v " + DEVICE_IMAGE_CACHE_DIR +
|
|
":" + "/mnt/images " + OPAE_IMG +
|
|
" fpgasupdate -y --log-level debug /mnt/images/" +
|
|
filename + " " + pci_addr)
|
|
|
|
# Issue the command to perform the firmware update.
|
|
subprocess.check_output(shlex.split(cmd),
|
|
stderr=subprocess.STDOUT)
|
|
# TODO: switch to subprocess.Popen, parse the output and send
|
|
# progress updates.
|
|
except subprocess.CalledProcessError as exc:
|
|
# Check the return code, send completion info to sysinv-conductor.
|
|
# "docker run" return code will be:
|
|
# 125 if the error is with Docker daemon itself
|
|
# 126 if the contained command cannot be invoked
|
|
# 127 if the contained command cannot be found
|
|
# Exit code of contained command otherwise
|
|
msg = ("Failed to update device image %s for device %s, "
|
|
"return code is %d, command output: %s." %
|
|
(filename, pci_addr, exc.returncode,
|
|
exc.output.decode('utf-8')))
|
|
LOG.error(msg)
|
|
LOG.error("Check for intel-max10 kernel logs.")
|
|
raise exception.SysinvException(msg)
|
|
|
|
|
|
def read_n3000_sysfs_file(pattern):
|
|
# Read a sysfs file related to the N3000.
|
|
# The result should be an empty string if the file doesn't exist,
|
|
# or a single line of text if it does.
|
|
|
|
# Convert the pattern to a list of matching filenames
|
|
filenames = glob(pattern)
|
|
|
|
# If there are no matching files, return an empty string.
|
|
if len(filenames) == 0:
|
|
return ""
|
|
|
|
# If there's more than one filename, complain.
|
|
if len(filenames) > 1:
|
|
LOG.warn("Pattern %s gave %s matching filenames, using the first." %
|
|
(pattern, len(filenames)))
|
|
|
|
filename = filenames[0]
|
|
infile = open(filename)
|
|
try:
|
|
line = infile.readline()
|
|
return line.strip()
|
|
except Exception:
|
|
LOG.exception("Unable to read file %s" % filename)
|
|
finally:
|
|
infile.close()
|
|
return ""
|
|
|
|
|
|
def get_n3000_root_hash(pci_addr):
|
|
# Query sysfs for the root key of the N3000 at the specified PCI address
|
|
root_key_pattern = (SYSFS_DEVICE_PATH + pci_addr + FME_PATH +
|
|
SPI_PATH + ROOT_HASH_PATH)
|
|
root_key = read_n3000_sysfs_file(root_key_pattern)
|
|
# If the root key hasn't been programmed, return an empty string.
|
|
if root_key == "hash not programmed":
|
|
root_key = ""
|
|
return root_key
|
|
|
|
|
|
def get_n3000_revoked_keys(pci_addr):
|
|
# Query sysfs for revoked keys of the N3000 at the specified PCI address
|
|
revoked_key_pattern = (SYSFS_DEVICE_PATH + pci_addr + FME_PATH +
|
|
SPI_PATH + CANCELLED_CSKS_PATH)
|
|
revoked_keys = read_n3000_sysfs_file(revoked_key_pattern)
|
|
return revoked_keys
|
|
|
|
|
|
def get_n3000_bitstream_id(pci_addr):
|
|
# Query sysfs for bitstream ID of the N3000 at the specified PCI address
|
|
bitstream_id_pattern = (SYSFS_DEVICE_PATH + pci_addr + FME_PATH +
|
|
BITSTREAM_ID_PATH)
|
|
bitstream_id = read_n3000_sysfs_file(bitstream_id_pattern)
|
|
return bitstream_id
|
|
|
|
|
|
def get_n3000_boot_page(pci_addr):
|
|
# Query sysfs for boot page of the N3000 at the specified PCI address
|
|
image_load_pattern = (SYSFS_DEVICE_PATH + pci_addr + FME_PATH +
|
|
SPI_PATH + IMAGE_LOAD_PATH)
|
|
image_load = read_n3000_sysfs_file(image_load_pattern)
|
|
if image_load == "0":
|
|
return "factory"
|
|
elif image_load == "1":
|
|
return "user"
|
|
else:
|
|
LOG.warn("Reading image load gave unexpected result: %s" % image_load)
|
|
return ""
|
|
|
|
|
|
def get_n3000_bmc_version(pci_addr, path):
|
|
version_pattern = (SYSFS_DEVICE_PATH + pci_addr + FME_PATH +
|
|
SPI_PATH + path)
|
|
version = read_n3000_sysfs_file(version_pattern)
|
|
|
|
# If we couldn't read the file, return an empty string.
|
|
if version == "":
|
|
return ""
|
|
|
|
# We're expecting a 32-bit value, possibly with "0x" in front.
|
|
try:
|
|
vint = int(version, 16)
|
|
except ValueError:
|
|
return ""
|
|
|
|
if vint >= 1 << 32:
|
|
LOG.warn("String (%s) read from file %s doesn't match the "
|
|
"expected pattern" % (version, version_pattern))
|
|
return ""
|
|
# There's probably a better way than this.
|
|
# We want to match the version that Intel's "fpgainfo" tool reports.
|
|
return ("%s.%s.%s.%s" % (chr(vint >> 24), str(vint >> 16 & 0xff),
|
|
str(vint >> 8 & 0xff), str(vint & 0xff)))
|
|
|
|
|
|
def get_n3000_bmc_fw_version(pci_addr):
|
|
return get_n3000_bmc_version(pci_addr, BMC_FW_VER_PATH)
|
|
|
|
|
|
def get_n3000_bmc_build_version(pci_addr):
|
|
return get_n3000_bmc_version(pci_addr, BMC_BUILD_VER_PATH)
|
|
|
|
|
|
def watchdog_action(action):
|
|
if action not in ["stop", "start"]:
|
|
LOG.warn("watchdog_action called with invalid action: %s", action)
|
|
return
|
|
try:
|
|
# Build up the command to perform the action.
|
|
cmd = ["systemctl", action, "hostw"]
|
|
|
|
# Issue the command to stop/start the watchdog
|
|
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
|
|
except subprocess.CalledProcessError as exc:
|
|
msg = ("Failed to %s hostw service, "
|
|
"return code is %d, command output: %s." %
|
|
(action, exc.returncode, exc.output))
|
|
LOG.warn(msg)
|
|
|
|
|
|
def stop_watchdog():
|
|
watchdog_action("stop")
|
|
|
|
|
|
def start_watchdog():
|
|
watchdog_action("start")
|
|
|
|
|
|
class FpgaAgentManager(service.PeriodicService):
|
|
"""Sysinv FPGA Agent service main class."""
|
|
|
|
RPC_API_VERSION = '1.0'
|
|
|
|
def __init__(self, host, topic):
|
|
serializer = objects_base.SysinvObjectSerializer()
|
|
super(FpgaAgentManager, self).__init__(host, topic, serializer=serializer)
|
|
|
|
self.host_uuid = None
|
|
|
|
def start(self):
|
|
super(FpgaAgentManager, self).start()
|
|
|
|
if os.path.isfile('/etc/sysinv/sysinv.conf'):
|
|
LOG.info('sysinv-fpga-agent started')
|
|
else:
|
|
LOG.info('No config file for sysinv-fpga-agent found.')
|
|
raise exception.ConfigNotFound(message="Unable to find sysinv config file!")
|
|
|
|
# Wait around until someone else updates the platform.conf file
|
|
# with our host UUID.
|
|
self.wait_for_host_uuid()
|
|
|
|
# Collect FPGA inventory and report to conductor at startup.
|
|
context = ctx.get_admin_context()
|
|
self.report_fpga_inventory(context)
|
|
|
|
def periodic_tasks(self, context, raise_on_error=False):
|
|
""" Periodic tasks are run at pre-specified intervals. """
|
|
return self.run_periodic_tasks(context, raise_on_error=raise_on_error)
|
|
|
|
def wait_for_host_uuid(self):
|
|
# Get our host UUID from /etc/platform/platform.conf. Note that the
|
|
# file can exist before the UUID is written to it.
|
|
prefix = "UUID="
|
|
while self.host_uuid is None:
|
|
if os.path.isfile(tsc.PLATFORM_CONF_FILE):
|
|
with open(tsc.PLATFORM_CONF_FILE, 'r') as platform_file:
|
|
for line in platform_file:
|
|
line = line.strip()
|
|
if not line.startswith(prefix):
|
|
continue
|
|
uuid = line[len(prefix):]
|
|
if uuidutils.is_uuid_like(uuid):
|
|
self.host_uuid = uuid
|
|
LOG.info("Agent found host UUID: %s" % uuid)
|
|
break
|
|
else:
|
|
LOG.info("UUID entry: %s in platform.conf "
|
|
"isn't uuid-like" % uuid)
|
|
|
|
time.sleep(5)
|
|
|
|
def report_fpga_inventory(self, context):
|
|
"""Collect FPGA data for this host.
|
|
|
|
This method allows host FPGA data to be collected.
|
|
|
|
:param: context: an admin context
|
|
:returns: nothing
|
|
"""
|
|
|
|
host_uuid = self.host_uuid
|
|
|
|
rpcapi = conductor_rpcapi.ConductorAPI(
|
|
topic=conductor_rpcapi.MANAGER_TOPIC)
|
|
|
|
fpgainfo_list = self.fpga_scan()
|
|
|
|
try:
|
|
LOG.info("reporting FPGA inventory for host %s: %s" %
|
|
(host_uuid, fpgainfo_list))
|
|
rpcapi.fpga_device_update_by_host(context, host_uuid, fpgainfo_list)
|
|
except exception.SysinvException:
|
|
LOG.exception("Exception updating fpga devices.")
|
|
pass
|
|
|
|
def fpga_scan(self):
|
|
|
|
# First get the PCI addresses of each supported FPGA device
|
|
cmd = ["lspci", "-Dm", "-d " + FPGA_VENDOR + ":" + FPGA_DEVICE]
|
|
|
|
try:
|
|
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
|
|
except subprocess.CalledProcessError as exc:
|
|
msg = ("Failed to get pci devices with vendor %s and device %s, "
|
|
"return code is %d, command output: %s." %
|
|
(FPGA_VENDOR, FPGA_DEVICE, exc.returncode, exc.output))
|
|
LOG.warn(msg)
|
|
raise exception.SysinvException(msg)
|
|
|
|
# Parse the output of the lspci command and grab the PCI address
|
|
fpga_addrs = []
|
|
for line in output.splitlines():
|
|
line = shlex.split(line.strip())
|
|
fpga_addrs.append(line[0])
|
|
|
|
fpgainfo_list = []
|
|
|
|
# Next, break down the PCI address into parts and use that to call the
|
|
# FPGA tools to get additional information
|
|
for addr in fpga_addrs:
|
|
# Store information for this FPGA
|
|
fpgainfo = {'pciaddr': addr}
|
|
fpgainfo['bmc_build_version'] = get_n3000_bmc_build_version(addr)
|
|
fpgainfo['bmc_fw_version'] = get_n3000_bmc_fw_version(addr)
|
|
fpgainfo['boot_page'] = get_n3000_boot_page(addr)
|
|
fpgainfo['bitstream_id'] = get_n3000_bitstream_id(addr)
|
|
fpgainfo['root_key'] = get_n3000_root_hash(addr)
|
|
fpgainfo['revoked_key_ids'] = get_n3000_revoked_keys(addr)
|
|
|
|
# TODO: Also retrieve the information about which NICs are on
|
|
# the FPGA device.
|
|
|
|
fpgainfo_list.append(fpgainfo)
|
|
|
|
return fpgainfo_list
|
|
|
|
def device_update_image(self, context, pci_addr, filename, transaction_id):
|
|
"""Write the device image to the device at the specified address.
|
|
|
|
Transaction is the transaction ID as specified by sysinv-conductor.
|
|
|
|
This must send back either success or failure to sysinv-conductor
|
|
via an RPC cast. The transaction ID is sent back to allow sysinv-conductor
|
|
to locate the transaction in the DB.
|
|
|
|
TODO: could get fancier with an image cache and delete based on LRU.
|
|
"""
|
|
|
|
rpcapi = conductor_rpcapi.ConductorAPI(
|
|
topic=conductor_rpcapi.MANAGER_TOPIC)
|
|
|
|
try:
|
|
LOG.info("ensure device image cache exists")
|
|
ensure_device_image_cache_exists()
|
|
|
|
# Pull the image from the controller.
|
|
LOG.info("fetch device image %s" % filename)
|
|
# For now, we only need to support AIO nodes, so just copy the
|
|
# file from where we know sysinv-conductor put it.
|
|
local_path = fetch_device_image_local(filename)
|
|
|
|
# TODO: when we need to support standalone workers, we'll need to
|
|
# pull in the image file via HTTP.
|
|
# local_path = fetch_device_image(filename)
|
|
|
|
# TODO: check CSK used to sign image, ensure it hasn't been cancelled
|
|
# TODO: check root key used to sign image, ensure it matches root key of hardware
|
|
# Note: may want to check these in the sysinv API too.
|
|
|
|
try:
|
|
LOG.info("setting transaction id %s as in progress" % transaction_id)
|
|
rpcapi.device_update_image_status(
|
|
context, self.host_uuid, transaction_id,
|
|
dconstants.DEVICE_IMAGE_UPDATE_IN_PROGRESS)
|
|
|
|
# Disable the watchdog service to prevent a reboot on things
|
|
# like critical process death. We don't want to reboot while
|
|
# flashing the FPGA.
|
|
stop_watchdog()
|
|
|
|
# Write the image to the specified PCI device.
|
|
# TODO: when we support more than just N3000, we'll need to
|
|
# pick the appropriate low-level write function based on the
|
|
# hardware type.
|
|
LOG.info("writing device image %s to device %s" % (filename, pci_addr))
|
|
write_device_image_n3000(filename, pci_addr)
|
|
|
|
# If we get an exception trying to send the status update
|
|
# there's not much we can do.
|
|
try:
|
|
LOG.info("setting transaction id %s as complete" % transaction_id)
|
|
rpcapi.device_update_image_status(
|
|
context, self.host_uuid, transaction_id,
|
|
dconstants.DEVICE_IMAGE_UPDATE_COMPLETED)
|
|
except Exception:
|
|
LOG.exception("Unable to send fpga update image status "
|
|
"completion message for transaction %s."
|
|
% transaction_id)
|
|
finally:
|
|
# Delete the image file.
|
|
os.remove(local_path)
|
|
# start the watchdog service again
|
|
start_watchdog()
|
|
|
|
except exception.SysinvException as exc:
|
|
LOG.info("setting transaction id %s as failed" % transaction_id)
|
|
rpcapi.device_update_image_status(context, self.host_uuid,
|
|
transaction_id,
|
|
dconstants.DEVICE_IMAGE_UPDATE_FAILED,
|
|
exc.message)
|