在之前的一次對裸金屬hypervisor資源同步問題的分析（https://www.cnblogs.com/wangwei1/p/13037517.html）中，關於裸金屬個數與nova側hypervisor個數不匹配的問題，當時懷疑是nova-compute服務的異常導致。後來對這個現象進行了進一步的分析，找出了最終的原因。

我們先分析下nova hypervisor-list獲取資料的流程

1、nova hypervisor-list 流程

在nova/api/openstack/compute/routes.py中，定義了hypervisors-list的路由，為hypervisors_controller中的index方法：

hypervisors_controller = functools.partial(
    _create_controller, hypervisors.HypervisorsController, [], [])

...

    ('/os-hypervisors', {
        'GET': [hypervisors_controller, 'index']
    }),

在nova/api/openstack/compute/hypervisors.py中，index的方法定義為：

    @wsgi.Controller.api_version(UUID_FOR_ID_MIN_VERSION)
    @validation.query_schema(hyper_schema.list_query_schema_v253,
                             UUID_FOR_ID_MIN_VERSION)
    @extensions.expected_errors((
 
400, 404))
    def index(self, req):
        limit, marker = common.get_limit_and_marker(req)
        return self._index(req, limit=limit, marker=marker, links=True)

其中，self._index方法定義為：

    def _index(self, req, limit=None, marker=None, links=False):
        return self._get_hypervisors(req, detail=False, limit=limit,
                                     marker
 
=marker, links=links)

而_get_hypervisors方法的定義如下：

    def _get_hypervisors(self, req, detail=False, limit=None, marker=None,
                         links=False):

            ...
            # Get all compute nodes.
            try:
                compute_nodes = self.host_api.compute_node_get_all(
                    context, limit=limit, marker=marker)
            except exception.MarkerNotFound:
                msg = _('marker [%s] not found') % marker
                raise webob.exc.HTTPBadRequest(explanation=msg)

        hypervisors_list = []
        for hyp in compute_nodes:
            try:
                instances = None
                if with_servers:
                    instances = self.host_api.instance_get_all_by_host(
                        context, hyp.host)
                service = self.host_api.service_get_by_compute_host(
                    context, hyp.host)
                hypervisors_list.append(
                    self._view_hypervisor(
                        hyp, service, detail, req, servers=instances))
            except (exception.ComputeHostNotFound,
                    exception.HostMappingNotFound):
                LOG.debug('Unable to find service for compute node %s. The '
                          'service may be deleted and compute nodes need to '
                          'be manually cleaned up.', hyp.host)

        hypervisors_dict = dict(hypervisors=hypervisors_list)
        if links:
            hypervisors_links = self._view_builder.get_links(
                req, hypervisors_list, detail)
            if hypervisors_links:
                hypervisors_dict['hypervisors_links'] = hypervisors_links
        return hypervisors_dict

其中，主要是通過 self.host_api.compute_node_get_all 獲取所有的hypervisor，並獲取每個hypervisor對應的service和instance資訊，最終通過self._view_hypervisor 進行展示。

方法 compute_node_get_all在nova/compute/api.py:HOSTAPI中定義如：

    def compute_node_get_all(self, context, limit=None, marker=None):
        load_cells()

        computes = []
        uuid_marker = marker and uuidutils.is_uuid_like(marker)
        for cell in CELLS:
            if cell.uuid == objects.CellMapping.CELL0_UUID:
                continue
            with nova_context.target_cell(context, cell) as cctxt:

                ...
                try:
                    cell_computes = objects.ComputeNodeList.get_by_pagination(
                        cctxt, limit=limit, marker=marker)
                except exception.MarkerNotFound:
                    continue
                computes.extend(cell_computes)
                ...
        return objects.ComputeNodeList(objects=computes)

這裡主要是通過方法objects.ComputeNodeList.get_by_pagination 獲取每個cell中的compute_nodes（hypervisors）資料。

而方法objects.ComputeNodeList.get_by_pagination 定義為：

    @base.remotable_classmethod
    def get_by_pagination(cls, context, limit=None, marker=None):
        db_computes = db.compute_node_get_all_by_pagination(
            context, limit=limit, marker=marker)
        return base.obj_make_list(context, cls(context), objects.ComputeNode,
                                  db_computes)

主要是通過db-api 中的compute_node_get_all_by_pagination 方法獲取資料。

在nova/db/sqlalchemy/api.py中，有如下定義：

@pick_context_manager_reader
def compute_node_get_all_by_pagination(context, limit=None, marker=None):
    return _compute_node_fetchall(context, limit=limit, marker=marker)

def _compute_node_fetchall(context, filters=None, limit=None, marker=None):
    select = _compute_node_select(context, filters, limit=limit, marker=marker)
    engine = get_engine(context=context)
    conn = engine.connect()

    results = conn.execute(select).fetchall()

    # Callers expect dict-like objects, not SQLAlchemy RowProxy objects...
    results = [dict(r) for r in results]
    conn.close()
    return results

對於_compute_node_select 方法，其定義為：

def _compute_node_select(context, filters=None, limit=None, marker=None):
    if filters is None:
        filters = {}

    cn_tbl = sa.alias(models.ComputeNode.__table__, name='cn')
    select = sa.select([cn_tbl])

    if context.read_deleted == "no":
        select = select.where(cn_tbl.c.deleted == 0)
    if "compute_id" in filters:
        select = select.where(cn_tbl.c.id == filters["compute_id"])
    if "service_id" in filters:
        select = select.where(cn_tbl.c.service_id == filters["service_id"])
    if "host" in filters:
        select = select.where(cn_tbl.c.host == filters["host"])
    if "hypervisor_hostname" in filters:
        hyp_hostname = filters["hypervisor_hostname"]
        select = select.where(cn_tbl.c.hypervisor_hostname == hyp_hostname)
    if "mapped" in filters:
        select = select.where(cn_tbl.c.mapped < filters['mapped'])
    if marker is not None:
        try:
            compute_node_get(context, marker)
        except exception.ComputeHostNotFound:
            raise exception.MarkerNotFound(marker=marker)
        select = select.where(cn_tbl.c.id > marker)
    if limit is not None:
        select = select.limit(limit)
    # Explicitly order by id, so we're not dependent on the native sort
    # order of the underlying DB.
    select = select.order_by(asc("id"))
    return select

所以，nova hypervisor-list就是從nova資料庫的compute_nodes 表中讀取資料。

2、裸金屬hypervisor同步流程分析

在nova/compute/manager.py中，有周期任務在執行hypervisor資源同步任務

    @periodic_task.periodic_task(spacing=CONF.update_resources_interval)
    def update_available_resource(self, context, startup=False):
        ...
        compute_nodes_in_db = self._get_compute_nodes_in_db(context,
                                                            use_slave=True,
                                                            startup=startup)
        nodenames = set(self.driver.get_available_nodes())
        for nodename in nodenames:
            self.update_available_resource_for_node(context, nodename)

        # Delete orphan compute node not reported by driver but still in db
        for cn in compute_nodes_in_db:
            if cn.hypervisor_hostname not in nodenames:
                LOG.info("Deleting orphan compute node %(id)s "
                         "hypervisor host is %(hh)s, "
                         "nodes are %(nodes)s",
                         {'id': cn.id, 'hh': cn.hypervisor_hostname,
                          'nodes': nodenames})
                cn.destroy()
                self.scheduler_client.reportclient.delete_resource_provider(
                    context, cn, cascade=True)

在這個週期任務中，先是呼叫_get_compute_nodes_in_db 從資料庫中讀取已有的compute_nodes資訊，然後通過self.driver.get_available_nodes() 從virtdriver中獲取對應的nodes資訊，接著遍歷virtdriver返回的nodes資訊，呼叫update_available_resource_for_node進行更新；

另外，還會遍歷資料庫中的compute_nodes資訊，如果某個compute_node不在virtdriver返回的資訊中，則將其從資料庫中移除掉：

        # Delete orphan compute node not reported by driver but still in db
        for cn in compute_nodes_in_db:
            if cn.hypervisor_hostname not in nodenames:
                LOG.info("Deleting orphan compute node %(id)s "
                         "hypervisor host is %(hh)s, "
                         "nodes are %(nodes)s",
                         {'id': cn.id, 'hh': cn.hypervisor_hostname,
                          'nodes': nodenames})
                cn.destroy()

在nova/virt/ironic/driver.py中，get_available_nodes方法定義為如下，主要是呼叫ironic-api介面，獲取nodes資訊。

    def get_available_nodes(self, refresh=False):
        self._refresh_cache()

        node_uuids = list(self.node_cache.keys())
        LOG.debug("Returning %(num_nodes)s available node(s)",
                  dict(num_nodes=len(node_uuids)))

        return node_uuids

根據virtdriver返回的nodes資訊，呼叫update_available_resource_for_node ，將node資訊轉換為對應的hypervisor資源，儲存到nova的資料庫中。

這裡主要是通過resource_tracker進行更新：

    def update_available_resource_for_node(self, context, nodename):
        rt = self._get_resource_tracker()
        try:
            rt.update_available_resource(context, nodename)
        except exception.ComputeHostNotFound:
            LOG.info("Compute node '%s' not found in "
                     "update_available_resource.", nodename)

            self._resource_tracker = None
            return
        except Exception:
            LOG.exception("Error updating resources for node %(node)s.",
                          {'node': nodename})

而resource_tracker物件的獲取，只在第一次生成ResourceTracker物件，後面用的都是這個物件：

    def _get_resource_tracker(self):
        if not self._resource_tracker:
            rt = resource_tracker.ResourceTracker(self.host, self.driver)
            self._resource_tracker = rt
        return self._resource_tracker

接著通過resource_tracker中的update_available_resource進行更新：

    def update_available_resource(self, context, nodename):
        LOG.debug("Auditing locally available compute resources for "
                  "%(host)s (node: %(node)s)",
                 {'node': nodename,
                  'host': self.host})
        resources = self.driver.get_available_resource(nodename)

        resources['host_ip'] = CONF.my_ip

        if "cpu_info" not in resources or resources["cpu_info"] is None:
            resources["cpu_info"] = ''

        self._verify_resources(resources)

        self._report_hypervisor_resource_view(resources)

        self._update_available_resource(context, resources)

先是根據nodename（這裡為node的uuid，是前面get_available_nodes時返回的uuid列表）呼叫virtdriver獲取對應的resource資訊。IronicDriver中的resource資訊主要是一些硬體屬性：

    def _node_resource(self, node):
        """Helper method to create resource dict from node stats."""
        properties = self._parse_node_properties(node)

        vcpus = properties['cpus']
        memory_mb = properties['memory_mb']
        local_gb = properties['local_gb']
        raw_cpu_arch = properties['raw_cpu_arch']
        cpu_arch = properties['cpu_arch']

        nodes_extra_specs = {}

        nodes_extra_specs['cpu_arch'] = raw_cpu_arch

        # ComputeCapabilitiesFilter
        capabilities = properties['capabilities']
        if capabilities:
            for capability in str(capabilities).split(','):
                parts = capability.split(':')
                if len(parts) == 2 and parts[0] and parts[1]:
                    nodes_extra_specs[parts[0].strip()] = parts[1]
                else:
                    LOG.warning("Ignoring malformed capability '%s'. "
                                "Format should be 'key:val'.", capability)

        vcpus_used = 0
        memory_mb_used = 0
        local_gb_used = 0

        if self._node_resources_used(node):
            vcpus_used = vcpus
            memory_mb_used = memory_mb
            local_gb_used = local_gb
        elif self._node_resources_unavailable(node):
            vcpus = 0
            memory_mb = 0
            local_gb = 0

        dic = {
            'hypervisor_hostname': str(node.uuid),
            'hypervisor_type': self._get_hypervisor_type(),
            'hypervisor_version': self._get_hypervisor_version(),
            'resource_class': node.resource_class,
            'cpu_info': None,
            'vcpus': vcpus,
            'vcpus_used': vcpus_used,
            'local_gb': local_gb,
            'local_gb_used': local_gb_used,
            'disk_available_least': local_gb - local_gb_used,
            'memory_mb': memory_mb,
            'memory_mb_used': memory_mb_used,
            'supported_instances': _get_nodes_supported_instances(cpu_arch),
            'stats': nodes_extra_specs,
            'numa_topology': None,
        }
        return dic

在update_available_resource中，接著是verify校驗、report進行列印，最後才是呼叫self._update_available_resource進行更新。

在_update_available_resource中，先是將resources 初始化為compute_node物件：

    def _init_compute_node(self, context, resources):
        nodename = resources['hypervisor_hostname']

        if nodename in self.compute_nodes:
            cn = self.compute_nodes[nodename]
            self._copy_resources(cn, resources)
            self._setup_pci_tracker(context, cn, resources)
            return

        cn = self._get_compute_node(context, nodename)
        if cn:
            self.compute_nodes[nodename] = cn
            self._copy_resources(cn, resources)
            self._setup_pci_tracker(context, cn, resources)
            return

        cn = objects.ComputeNode(context)
        cn.host = self.host
        self._copy_resources(cn, resources)
        self.compute_nodes[nodename] = cn
        cn.create()
        LOG.info('Compute node record created for '
                 '%(host)s:%(node)s with uuid: %(uuid)s',
                 {'host': self.host, 'node': nodename, 'uuid': cn.uuid})

        self._setup_pci_tracker(context, cn, resources)