Nova attach volume的流程分析
阿新 • • 發佈:2019-02-05
作為個人學習筆記分享,有任何問題歡迎交流!
(2013.7.5更新)
Nova中volume掛載流程分為兩部分:掛載命令的傳送和接收處理
1 掛載命令的傳送
1.1提供API介面
程式碼來源:nova/api/openstack/contrib/volumes.py:VolumeAttachmentController.create():
@wsgi.serializers(xml=VolumeAttachmentTemplate) def create(self, req, server_id, body): """Attach a volume to an instance.""" context = req.environ['nova.context'] authorize(context) authorize_attach(context, action='create') if not self.is_valid_body(body, 'volumeAttachment'): raise exc.HTTPUnprocessableEntity() #從請求中獲取卷ID和裝置名稱 volume_id = body['volumeAttachment']['volumeId'] device = body['volumeAttachment'].get('device') self._validate_volume_id(volume_id) msg = _("Attach volume %(volume_id)s to instance %(server_id)s" " at %(device)s") % locals() LOG.audit(msg, context=context) try: instance = self.compute_api.get(context, server_id)#compute_api=compute.API(),compute.API()即為/nova/compute/__init__.py:API(*args,**kwargs),該函式匯入了一個類,被匯入的類是由_compute_opts中compute_api_class決定的,compute_api_class的預設值為:nova.compute.api.API,所以get()既是nova/compute/api.py:API.get(),根據例項ID獲取一個例項。 device = self.compute_api.attach_volume(context, instance, volume_id, device)#掛載一個存在的捲到一個存在的例項 except exception.NotFound: raise exc.HTTPNotFound() except exception.InstanceInvalidState as state_error: common.raise_http_conflict_for_instance_invalid_state(state_error, 'attach_volume')
1.2傳送RPCcast非同步請求,給instance掛載上volume_id,掛載點在device.
程式碼來源:nova/compute/api.py:API.attach_volume(context, instance, volume_id,device)
#volume_api=nova/volume/cinder/api.py:API try: volume = self.volume_api.get(context, volume_id)通過cinderclient從cinder獲取volume. self.volume_api.check_attach(context, volume, instance=instance)#檢查volume是否可用 self.volume_api.reserve_volume(context, volume) self.compute_rpcapi.attach_volume(context, instance=instance, volume_id=volume_id, mountpoint=device)#compute_rpcapi=nova/compute/rpcapi.py:ComputeAPI except Exception: with excutils.save_and_reraise_exception(): self.db.block_device_mapping_destroy_by_instance_and_device( context, instance['uuid'], device) #nova/compute/rpcapi.py:ComputeAPI.attach_volume(): def attach_volume(self, ctxt, instance, volume_id, mountpoint): instance_p = jsonutils.to_primitive(instance) self.cast(ctxt, self.make_msg('attach_volume',#cast呼叫 instance=instance_p, volume_id=volume_id, mountpoint=mountpoint), topic=_compute_topic(self.topic, ctxt, None, instance))
(2013.7.9更新)
2.掛載命令的接收處理
2.1 Cast呼叫的接收端
掛載命令由nova-compute服務負責處理,程式碼來源:nova/compute/manager.py:ComputeManager.attach_volume()
#attach_volume()呼叫_attach_volume(): def _attach_volume(self, context, volume_id, mountpoint, instance): volume = self.volume_api.get(context, volume_id) context = context.elevated() LOG.audit(_('Attaching volume %(volume_id)s to %(mountpoint)s'), locals(), context=context, instance=instance) try: connector = self.driver.get_volume_connector(instance) #driver=nova/virt/libvirt/driver.py:LibvirtDriver, driver.get_volume_connector(),獲取一個iscsi initiator connection_info = self.volume_api.initialize_connection(context, volume, connector)#volume_api=nova/volume/cinder.py:API,通過cinderclient呼叫iscsi的initialize_connection初始化initiator except Exception: # pylint: disable=W0702 with excutils.save_and_reraise_exception(): msg = _("Failed to connect to volume %(volume_id)s " "while attaching at %(mountpoint)s") LOG.exception(msg % locals(), context=context, instance=instance) self.volume_api.unreserve_volume(context, volume) if 'serial' not in connection_info: connection_info['serial'] = volume_id try: self.driver.attach_volume(connection_info,#掛載卷 instance, mountpoint) #nova/virt/libvirt/driver.py:LibvirtDriver.attach_volume() except Exception: # pylint: disable=W0702 with excutils.save_and_reraise_exception(): msg = _("Failed to attach volume %(volume_id)s " "at %(mountpoint)s") LOG.exception(msg % locals(), context=context, instance=instance) self.volume_api.terminate_connection(context, volume, connector) #update volume's database status in cinder through cinderclient self.volume_api.attach(context, volume, instance['uuid'], mountpoint)
2.2 掛載卷
程式碼來源:nova/virt/libvirt/driver.py:LibvirtDriver.attach_volume()
def attach_volume(self, connection_info, instance, mountpoint):
instance_name = instance['name']
virt_dom = self._lookup_by_name(instance_name)
disk_dev = mountpoint.rpartition("/")[2]
disk_info = {
'dev': disk_dev,
'bus': blockinfo.get_disk_bus_for_disk_dev(CONF.libvirt_type,
disk_dev),
'type': 'disk',
}
conf = self.volume_driver_method('connect_volume',
connection_info,
disk_info)#根據driver的型別呼叫相應的connect_volume方法,這裡將呼叫nova/virt/libvirt/volume.py:LibvirtISCSIVolumeDriver.connect_volume(),connect_volume方法完成的工作是在發現target,見下面。
self.set_cache_mode(conf)
try:
# NOTE(vish): We can always affect config because our
# domains are persistent, but we should only
# affect live if the domain is running.
##LOG.info(_('attach_volume action is here! ozg.log'))
flags = libvirt.VIR_DOMAIN_AFFECT_CONFIG #flags=3
state = LIBVIRT_POWER_STATE[virt_dom.info()[0]]
if state == power_state.RUNNING:
flags |= libvirt.VIR_DOMAIN_AFFECT_LIVE
virt_dom.attachDeviceFlags(conf.to_xml(), flags)"""conf.to_xml()的值為:<disk type="block" device="disk">
<driver name="qemu" type="raw" cache="none"/>
<source dev="/dev/disk/by-path/ip-192.168.88.168:3260-iscsi-iqn.2010-10.org.openstack:volume-c7a768b1-5b4a-49c3-80fe-a1ef007c52c1-lun-1"/>
<target bus="virtio" dev="vdc"/>
<serial>c7a768b1-5b4a-49c3-80fe-a1ef007c52c1</serial>
</disk>,attachDeviceFlags():建立一個虛擬裝置並掛載到後端"""
except Exception, ex:
if isinstance(ex, libvirt.libvirtError):
errcode = ex.get_error_code()
if errcode == libvirt.VIR_ERR_OPERATION_FAILED:
self.volume_driver_method('disconnect_volume',
connection_info,
disk_dev)
raise exception.DeviceIsBusy(device=disk_dev)
with excutils.save_and_reraise_exception():
self.volume_driver_method('disconnect_volume',
connection_info,
disk_dev)
2.3發現target
程式碼來源:/nova/virt/libvirt/volume.py:LibvirtISCSIVolumeDriver.
connect_volume()
@lockutils.synchronized('connect_volume', 'nova-')
def connect_volume(self, connection_info, disk_info):
"""Attach the volume to instance_name."""
conf = super(LibvirtISCSIVolumeDriver,
self).connect_volume(connection_info,
disk_info)
iscsi_properties = connection_info['data']
#Multipath用來實現裝置的持久化和多路徑訪問
libvirt_iscsi_use_multipath = CONF.libvirt_iscsi_use_multipath
if libvirt_iscsi_use_multipath:#從配置檔案中判斷是否支援multipath
#multipath installed, discovering other targets if available
#multipath should be configured on the nova-compute node,
#in order to fit storage vendor
out = self._run_iscsiadm_bare(['-m',
'discovery',
'-t',
'sendtargets',
'-p',
iscsi_properties['target_portal']],
check_exit_code=[0, 255])[0] \
or ""
for ip in self._get_target_portals_from_iscsiadm_output(out):
props = iscsi_properties.copy()
props['target_portal'] = ip
self._connect_to_iscsi_portal(props)
self._rescan_iscsi()
else:
self._connect_to_iscsi_portal(iscsi_properties)#連線iscsi,見下面
host_device = ("/dev/disk/by-path/ip-%s-iscsi-%s-lun-%s" %
(iscsi_properties['target_portal'],
iscsi_properties['target_iqn'],
iscsi_properties.get('target_lun', 0)))
# The /dev/disk/by-path/... node is not always present immediately
# TODO(justinsb): This retry-with-delay is a pattern, move to utils?
tries = 0
disk_dev = disk_info['dev']
while not os.path.exists(host_device):
if tries >= CONF.num_iscsi_scan_tries:
raise exception.NovaException(_("iSCSI device not found at %s")
% (host_device))
LOG.warn(_("ISCSI volume not yet found at: %(disk_dev)s. "
"Will rescan & retry. Try number: %(tries)s") %
locals())
# The rescan isn't documented as being necessary(?), but it helps
self._run_iscsiadm(iscsi_properties, ("--rescan",))#重新掃描
tries = tries + 1#重試tries次,預設3次
if not os.path.exists(host_device):
time.sleep(tries ** 2)
if tries != 0:
LOG.debug(_("Found iSCSI node %(disk_dev)s "
"(after %(tries)s rescans)") %
locals())
if libvirt_iscsi_use_multipath:
#we use the multipath device instead of the single path device
self._rescan_multipath()
multipath_device = self._get_multipath_device_name(host_device)
if multipath_device is not None:
host_device = multipath_device
conf.source_type = "block"
conf.source_path = host_device
return conf
程式碼來源:nova/virt/libvirt/volume.py: LibvirtISCSIVolumeDriver
def _connect_to_iscsi_portal(self, iscsi_properties):
# NOTE(vish): If we are on the same host as nova volume, the
# discovery makes the target so we don't need to
# run --op new. Therefore, we check to see if the
# target exists, and if we get 255 (Not Found), then
# we run --op new. This will also happen if another
# volume is using the same target.
try:
self._run_iscsiadm(iscsi_properties, ())
#執行:iscsiadm -m node -T iqn.2010-10.org.openstack:volume-c7a768b1-5b4a-49c3-80fe-a1ef007c52c1 -p 192.168.88.168:3260 execute /usr/lib/python2.7/dist-packages/nova/utils.py:208
except exception.ProcessExecutionError as exc:
# iscsiadm returns 21 for "No records found" after version 2.0-871
if exc.exit_code in [21, 255]:
self._run_iscsiadm(iscsi_properties, ('--op', 'new'))
else:
raise
if iscsi_properties.get('auth_method'):#get()返回CHAP
self._iscsiadm_update(iscsi_properties,
"node.session.auth.authmethod",
iscsi_properties['auth_method'])
#執行:iscsiadm -m node -T iqn.2010-10.org.openstack:volume-c7a768b1-5b4a-49c3-80fe-a1ef007c52c1 -p 192.168.88.168:3260 --op update -n node.session.auth.authmethod -v CHAP execute /usr/lib/python2.7/dist-packages/nova/utils.py:208
self._iscsiadm_update(iscsi_properties,
"node.session.auth.username",
iscsi_properties['auth_username'])
#執行:iscsiadm -m node -T iqn.2010-10.org.openstack:volume-c7a768b1-5b4a-49c3-80fe-a1ef007c52c1 -p 192.168.88.168:3260 --op update -n node.session.auth.username -v AREmd94wksBpTxEcJ5Yh execute /usr/lib/python2.7/dist-packages/nova/utils.py:208
self._iscsiadm_update(iscsi_properties,
"node.session.auth.password",
iscsi_properties['auth_password'])
#執行:iscsiadm -m node -T iqn.2010-10.org.openstack:volume-c7a768b1-5b4a-49c3-80fe-a1ef007c52c1 -p 192.168.88.168:3260 --op update -n node.session.auth.password -v wV4WuDvACoVyYA9Ru5tz execute /usr/lib/python2.7/dist-packages/nova/utils.py:208
#duplicate logins crash iscsiadm after load,
#so we scan active sessions to see if the node is logged in.
out = self._run_iscsiadm_bare(["-m", "session"],
run_as_root=True,
check_exit_code=[0, 1, 21])[0] or ""
#執行:iscsiadm ['-m', 'session']: stdout=
tcp: [20] 192.168.88.168:3260,1
iqn.2010-10.org.openstack:volume-14abe479-830d-4661-8047-a49414970f67
tcp: [4] 192.168.88.168:3260,1 iqn.2010-10.org.openstack:volume-993d0ed4-78b2-4c9f-a881-6aacafa173eb
stderr= _run_iscsiadm_bare /usr/lib/python2.7/dist-packages/nova/virt/libvirt/volume.py:423
portals = [{'portal': p.split(" ")[2], 'iqn': p.split(" ")[3]}
for p in out.splitlines() if p.startswith("tcp:")]
# portals=[{'iqn': 'iqn.2010-10.org.openstack:volume-14abe479-830d-4661-8047-a49414970f67', 'portal': '192.168.88.168:3260,1'}, {'iqn': 'iqn.2010-10.org.openstack:volume-993d0ed4-78b2-4c9f-a881-6aacafa173eb', 'portal': '192.168.88.168:3260,1'}]
stripped_portal = iscsi_properties['target_portal'].split(",")[0]
if len(portals) == 0 or len([s for s in portals
if stripped_portal ==
s['portal'].split(",")[0]
and
s['iqn'] ==
iscsi_properties['target_iqn']]
) == 0:
try:
self._run_iscsiadm(iscsi_properties,
("--login",),
check_exit_code=[0, 255])
#執行:iscsiadm -m node -T iqn.2010-10.org.openstack:volume-c7a768b1-5b4a-49c3-80fe-a1ef007c52c1 -p 192.168.88.168:3260 --login execute /usr/lib/python2.7/dist-packages/nova/utils.py:208
except exception.ProcessExecutionError as err:
#as this might be one of many paths,
#only set successfull logins to startup automatically
if err.exit_code in [15]:
self._iscsiadm_update(iscsi_properties,
"node.startup",
"automatic")
#執行:iscsiadm -m node -T iqn.2010-10.org.openstack:volume-c7a768b1-5b4a-49c3-80fe-a1ef007c52c1 -p 192.168.88.168:3260 --op update -n node.startup -v automatic execute /usr/lib/python2.7/dist-packages/nova/utils.py:208
return
self._iscsiadm_update(iscsi_properties,
"node.startup",
"automatic")