Coverage for libs/sm/SR.py: 56%

2# Copyright (C) Citrix Systems Inc.

4# This program is free software; you can redistribute it and/or modify

5# it under the terms of the GNU Lesser General Public License as published

6# by the Free Software Foundation; version 2.1 only.

8# This program is distributed in the hope that it will be useful,

9# but WITHOUT ANY WARRANTY; without even the implied warranty of

10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

11# GNU Lesser General Public License for more details.

12#

13# You should have received a copy of the GNU Lesser General Public License

14# along with this program; if not, write to the Free Software Foundation, Inc.,

15# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

16#

17# SR: Base class for storage repositories

18#

20from sm import VDI

21import xml.dom.minidom

22from sm.core import xs_errors

23import XenAPI # pylint: disable=import-error

24import xmlrpc.client

25from sm.core import util

26import copy

27import os

28import traceback

30MOUNT_BASE = '/run/sr-mount'

31DEFAULT_TAP = 'vhd'

32TAPDISK_UTIL = '/usr/sbin/td-util'

33MASTER_LVM_CONF = '/etc/lvm/master'

35# LUN per VDI key for XenCenter

36LUNPERVDI = "LUNperVDI"

42def deviceCheck(op):

43 def wrapper(self, *args):

44 if 'device' not in self.dconf:

45 raise xs_errors.XenError('ConfigDeviceMissing')

46 return op(self, *args)

47 return wrapper

50backends = []

53def registerSR(SRClass):

54 """Register SR with handler. All SR subclasses should call this in

55 the module file

56 """

57 backends.append(SRClass)

60def driver(type):

61 """Find the SR for the given dconf string"""

62 for d in backends: 62 ↛ 65line 62 didn't jump to line 65, because the loop on line 62 didn't complete

63 if d.handles(type):

64 return d

65 raise xs_errors.XenError('SRUnknownType')

68class SR(object):

69 """Semi-abstract storage repository object.

71 Attributes:

72 uuid: string, UUID

73 label: string

74 description: string

75 vdis: dictionary, VDI objects indexed by UUID

76 physical_utilisation: int, bytes consumed by VDIs

77 virtual_allocation: int, bytes allocated to this repository (virtual)

78 physical_size: int, bytes consumed by this repository

79 sr_vditype: string, repository type

80 """

82 def handles(type):

83 """Returns True if this SR class understands the given dconf string"""

84 return False

85 handles = staticmethod(handles)

87 def __init__(self, srcmd, sr_uuid):

88 """Base class initializer. All subclasses should call SR.__init__

89 in their own

90 initializers.

92 Arguments:

93 srcmd: SRCommand instance, contains parsed arguments

94 """

95 try:

96 self.other_config = {}

97 self.srcmd = srcmd

98 self.dconf = srcmd.dconf

99 if 'session_ref' in srcmd.params:

100 self.session_ref = srcmd.params['session_ref']

101 self.session = XenAPI.xapi_local()

102 self.session._session = self.session_ref

103 if 'subtask_of' in self.srcmd.params: 103 ↛ 104line 103 didn't jump to line 104, because the condition on line 103 was never true

104 self.session.transport.add_extra_header('Subtask-of', self.srcmd.params['subtask_of'])

105 else:

106 self.session = None

107

108 if 'host_ref' not in self.srcmd.params:

109 self.host_ref = ""

110 else:

111 self.host_ref = self.srcmd.params['host_ref']

112

113 self.sr_ref = self.srcmd.params.get('sr_ref')

114

115 if 'device_config' in self.srcmd.params:

116 if self.dconf.get("SRmaster") == "true":

117 os.environ['LVM_SYSTEM_DIR'] = MASTER_LVM_CONF

118

119 if 'device_config' in self.srcmd.params:

120 if 'SCSIid' in self.srcmd.params['device_config']:

121 dev_path = '/dev/disk/by-scsid/' + self.srcmd.params['device_config']['SCSIid']

122 os.environ['LVM_DEVICE'] = dev_path

123 util.SMlog('Setting LVM_DEVICE to %s' % dev_path)

124

125 except TypeError:

126 raise Exception(traceback.format_exc())

127 except Exception as e:

128 raise e

129 raise xs_errors.XenError('SRBadXML')

130

131 self.uuid = sr_uuid

132

133 self.label = ''

134 self.description = ''

135 self.cmd = srcmd.params['command']

136 self.vdis = {}

137 self.physical_utilisation = 0

138 self.virtual_allocation = 0

139 self.physical_size = 0

140 self.sr_vditype = ''

141 self.passthrough = False

142 # XXX: if this is really needed then we must make a deep copy

143 self.original_srcmd = copy.deepcopy(self.srcmd)

144 self.default_vdi_visibility = True

145 self.scheds = ['none', 'noop']

146 self._mpathinit()

147 self.direct = False

148 self.ops_exclusive = []

149 self.driver_config = {}

150

151 self.load(sr_uuid)

152

153 @staticmethod

154 def from_uuid(session, sr_uuid):

155 import importlib.machinery

156 import types

157

158 _SR = session.xenapi.SR

159 sr_ref = _SR.get_by_uuid(sr_uuid)

160 sm_type = _SR.get_type(sr_ref)

161 # NB. load the SM driver module

162

163 _SM = session.xenapi.SM

164 sms = _SM.get_all_records_where('field "type" = "%s"' % sm_type)

165 sm_ref, sm = sms.popitem()

166 assert not sms

167

168 driver_path = _SM.get_driver_filename(sm_ref)

169 driver_real = os.path.realpath(driver_path)

170 module_name = os.path.basename(driver_path)

171

172 loader = importlib.machinery.SourceFileLoader(module_name, driver_real)

173 module = types.ModuleType(loader.name)

174 loader.exec_module(module)

175 target = driver(sm_type)

176 # NB. get the host pbd's device_config

177

178 host_ref = util.get_localhost_ref(session)

179

180 _PBD = session.xenapi.PBD

181 pbds = _PBD.get_all_records_where('field "SR" = "%s" and' % sr_ref +

182 'field "host" = "%s"' % host_ref)

183 pbd_ref, pbd = pbds.popitem()

184 assert not pbds

185

186 device_config = _PBD.get_device_config(pbd_ref)

187 # NB. make srcmd, to please our supersized SR constructor.

188 # FIXME

189

190 from sm.SRCommand import SRCommand

191 # pylint: disable=E1101

192 cmd = SRCommand(module.DRIVER_INFO)

193 cmd.dconf = device_config

194 cmd.params = {'session_ref': session._session,

195 'host_ref': host_ref,

196 'device_config': device_config,

197 'sr_ref': sr_ref,

198 'sr_uuid': sr_uuid,

199 'command': 'nop'}

200

201 return target(cmd, sr_uuid)

202

203 def block_setscheduler(self, dev):

204 try:

205 realdev = os.path.realpath(dev)

206 disk = util.diskFromPartition(realdev)

207

208 # the normal case: the sr default scheduler (typically none/noop),

209 # potentially overridden by SR.other_config:scheduler

210 other_config = self.session.xenapi.SR.get_other_config(self.sr_ref)

211 sched = other_config.get('scheduler')

212 if not sched or sched in self.scheds: 212 ↛ 213line 212 didn't jump to line 213, because the condition on line 212 was never true

213 scheds = self.scheds

214 else:

215 scheds = [sched]

216

217 # special case: BFQ/CFQ if the underlying disk holds dom0's file systems.

218 if disk in util.dom0_disks(): 218 ↛ 219, 218 ↛ 2212 missed branches: 1) line 218 didn't jump to line 219, because the condition on line 218 was never true, 2) line 218 didn't jump to line 221, because the condition on line 218 was never false

219 scheds = ['bfq', 'cfq']

220

221 util.SMlog("Block scheduler: %s (%s) wants %s" % (dev, disk, scheds))

222 util.set_scheduler(realdev[5:], scheds)

223 except Exception as e:

224 util.SMlog("Failed to set block scheduler on %s: %s" % (dev, e))

225

226 def _addLUNperVDIkey(self):

227 try:

228 self.session.xenapi.SR.add_to_sm_config(self.sr_ref, LUNPERVDI, "true")

229 except:

230 pass

231

232 def create(self, uuid, size):

233 """Create this repository.

234 This operation may delete existing data.

235

236 The operation is NOT idempotent. The operation will fail

237 if an SR of the same UUID and driver type already exits.

238

239 Returns:

240 None

241 Raises:

242 SRUnimplementedMethod

243 """

244 raise xs_errors.XenError('Unimplemented')

245

246 def delete(self, uuid):

247 """Delete this repository and its contents.

248

249 This operation IS idempotent -- it will succeed if the repository

250 exists and can be deleted or if the repository does not exist.

251 The caller must ensure that all VDIs are deactivated and detached

252 and that the SR itself has been detached before delete().

253 The call will FAIL if any VDIs in the SR are in use.

254

255 Returns:

256 None

257 Raises:

258 SRUnimplementedMethod

259 """

260 raise xs_errors.XenError('Unimplemented')

261

262 def update(self, uuid):

263 """Refresh the fields in the SR object

264

265 Returns:

266 None

267 Raises:

268 SRUnimplementedMethod

269 """

270 # no-op unless individual backends implement it

271 return

272

273 def attach(self, uuid):

274 """Initiate local access to the SR. Initialises any

275 device state required to access the substrate.

276

277 Idempotent.

278

279 Returns:

280 None

281 Raises:

282 SRUnimplementedMethod

283 """

284 raise xs_errors.XenError('Unimplemented')

285

286 def after_master_attach(self, uuid):

287 """Perform actions required after attaching on the pool master

288 Return:

289 None

290 """

291 try:

292 self.scan(uuid)

293 except Exception as e:

294 util.SMlog("Error in SR.after_master_attach %s" % e)

295 msg_name = "POST_ATTACH_SCAN_FAILED"

296 msg_body = "Failed to scan SR %s after attaching, " \

297 "error %s" % (uuid, e)

298 self.session.xenapi.message.create(

299 msg_name, 2, "SR", uuid, msg_body)

300

301 def detach(self, uuid):

302 """Remove local access to the SR. Destroys any device

303 state initiated by the sr_attach() operation.

304

305 Idempotent. All VDIs must be detached in order for the operation

306 to succeed.

307

308 Returns:

309 None

310 Raises:

311 SRUnimplementedMethod

312 """

313 raise xs_errors.XenError('Unimplemented')

314

315 def probe(self):

316 """Perform a backend-specific scan, using the current dconf. If the

317 dconf is complete, then this will return a list of the SRs present of

318 this type on the device, if any. If the dconf is partial, then a

319 backend-specific scan will be performed, returning results that will

320 guide the user in improving the dconf.

321

322 Idempotent.

323

324 xapi will ensure that this is serialised wrt any other probes, or

325 attach or detach operations on this host.

326

327 Returns:

328 An XML fragment containing the scan results. These are specific

329 to the scan being performed, and the current backend.

330 Raises:

331 SRUnimplementedMethod

332 """

333 raise xs_errors.XenError('Unimplemented')

334

335 def scan(self, uuid):

336 """

337 Returns:

338 """

339 # Update SR parameters

340 self._db_update()

341 # Synchronise VDI list

342 scanrecord = ScanRecord(self)

343 scanrecord.synchronise()

344

345 def replay(self, uuid):

346 """Replay a multi-stage log entry

347

348 Returns:

349 None

350 Raises:

351 SRUnimplementedMethod

352 """

353 raise xs_errors.XenError('Unimplemented')

354

355 def content_type(self, uuid):

356 """Returns the 'content_type' of an SR as a string"""

357 return xmlrpc.client.dumps((str(self.sr_vditype), ), "", True)

358

359 def load(self, sr_uuid):

360 """Post-init hook"""

361 pass

362

363 def check_sr(self, sr_uuid):

364 """Hook to check SR health"""

365 pass

366

367 def vdi(self, uuid):

368 """Return VDI object owned by this repository"""

369 if uuid not in self.vdis:

370 self.vdis[uuid] = VDI.VDI(self, uuid)

371 raise xs_errors.XenError('Unimplemented')

372 return self.vdis[uuid]

373

374 def forget_vdi(self, uuid):

375 vdi = self.session.xenapi.VDI.get_by_uuid(uuid)

376 self.session.xenapi.VDI.db_forget(vdi)

377

378 def cleanup(self):

379 # callback after the op is done

380 pass

381

382 def _db_update(self):

383 sr = self.session.xenapi.SR.get_by_uuid(self.uuid)

384 self.session.xenapi.SR.set_virtual_allocation(sr, str(self.virtual_allocation))

385 self.session.xenapi.SR.set_physical_size(sr, str(self.physical_size))

386 self.session.xenapi.SR.set_physical_utilisation(sr, str(self.physical_utilisation))

387

388 def _toxml(self):

389 dom = xml.dom.minidom.Document()

390 element = dom.createElement("sr")

391 dom.appendChild(element)

392

393 # Add default uuid, physical_utilisation, physical_size and

394 # virtual_allocation entries

395 for attr in ('uuid', 'physical_utilisation', 'virtual_allocation',

396 'physical_size'):

397 try:

398 aval = getattr(self, attr)

399 except AttributeError:

400 raise xs_errors.XenError(

401 'InvalidArg', opterr='Missing required field [%s]' % attr)

402

403 entry = dom.createElement(attr)

404 element.appendChild(entry)

405 textnode = dom.createTextNode(str(aval))

406 entry.appendChild(textnode)

407

408 # Add the default_vdi_visibility entry

409 entry = dom.createElement('default_vdi_visibility')

410 element.appendChild(entry)

411 if not self.default_vdi_visibility:

412 textnode = dom.createTextNode('False')

413 else:

414 textnode = dom.createTextNode('True')

415 entry.appendChild(textnode)

416

417 # Add optional label and description entries

418 for attr in ('label', 'description'):

419 try:

420 aval = getattr(self, attr)

421 except AttributeError:

422 continue

423 if aval:

424 entry = dom.createElement(attr)

425 element.appendChild(entry)

426 textnode = dom.createTextNode(str(aval))

427 entry.appendChild(textnode)

428

429 # Create VDI sub-list

430 if self.vdis:

431 for uuid in self.vdis:

432 if not self.vdis[uuid].deleted:

433 vdinode = dom.createElement("vdi")

434 element.appendChild(vdinode)

435 self.vdis[uuid]._toxml(dom, vdinode)

436

437 return dom

438

439 def _fromxml(self, str, tag):

440 dom = xml.dom.minidom.parseString(str)

441 objectlist = dom.getElementsByTagName(tag)[0]

442 taglist = {}

443 for node in objectlist.childNodes:

444 taglist[node.nodeName] = ""

445 for n in node.childNodes:

446 if n.nodeType == n.TEXT_NODE:

447 taglist[node.nodeName] += n.data

448 return taglist

449

450 def _splitstring(self, str):

451 elementlist = []

452 for i in range(0, len(str)):

453 elementlist.append(str[i])

454 return elementlist

455

456 def _mpathinit(self):

457 self.mpath = "false"

458 try:

459 if 'multipathing' in self.dconf and \ 459 ↛ 461line 459 didn't jump to line 461, because the condition on line 459 was never true

460 'multipathhandle' in self.dconf:

461 self.mpath = self.dconf['multipathing']

462 self.mpathhandle = self.dconf['multipathhandle']

463 else:

464 hconf = self.session.xenapi.host.get_other_config(self.host_ref)

465 self.mpath = hconf['multipathing']

466 self.mpathhandle = hconf.get('multipathhandle', 'dmp')

467

468 if self.mpath != "true": 468 ↛ 474line 468 didn't jump to line 474, because the condition on line 468 was never false

469 self.mpath = "false"

470 self.mpathhandle = "null"

471

472 # Attempt to import the specified mpath module and return None

473 # if this does not throw an exception.

474 module_name = "sm.core.mpath_%s" % self.mpathhandle

475 self.mpathmodule = __import__(module_name, fromlist=[None])

476 return None

477 except:

478 # If attempting to import the specified mpath module failed for any reason,

479 # repeat with the "null" mpath module. Any exception this throws is the caller's

480 # problem.

481 self.mpath = "false"

482 self.mpathhandle = "null"

483 module_name = "sm.core.mpath_%s" % self.mpathhandle

484 self.mpathmodule = __import__(module_name, fromlist=[None])

485 return None

486

487 def _mpathHandle(self):

488 if self.mpath == "true": 488 ↛ 489line 488 didn't jump to line 489, because the condition on line 488 was never true

489 self.mpathmodule.activate()

490 else:

491 self.mpathmodule.deactivate()

492

493 def _pathrefresh(self, obj):

494 SCSIid = getattr(self, 'SCSIid')

495 self.dconf['device'] = self.mpathmodule.path(SCSIid)

496 super(obj, self).load(self.uuid)

497

498 def _setMultipathableFlag(self, SCSIid=''):

499 try:

500 sm_config = self.session.xenapi.SR.get_sm_config(self.sr_ref)

501 sm_config['multipathable'] = 'true'

502 self.session.xenapi.SR.set_sm_config(self.sr_ref, sm_config)

503

504 if self.mpath == "true" and len(SCSIid): 504 ↛ 505line 504 didn't jump to line 505, because the condition on line 504 was never true

505 util.kickpipe_mpathcount()

506 except:

507 pass

508

509 def check_dconf(self, key_list, raise_flag=True):

510 """ Checks if all keys in 'key_list' exist in 'self.dconf'.

511

512 Input:

513 key_list: a list of keys to check if they exist in self.dconf

514 raise_flag: if true, raise an exception if there are 1 or more

515 keys missing

516

517 Return: set() containing the missing keys (empty set() if all exist)

518 Raise: xs_errors.XenError('ConfigParamsMissing')

519 """

520

521 missing_keys = {key for key in key_list if key not in self.dconf}

522

523 if missing_keys and raise_flag:

524 errstr = 'device-config is missing the following parameters: ' + \

525 ', '.join([key for key in missing_keys])

526 raise xs_errors.XenError('ConfigParamsMissing', opterr=errstr)

527

528 return missing_keys

529

530

531class ScanRecord:

532 def __init__(self, sr):

533 self.sr = sr

534 self.__xenapi_locations = {}

535 self.__xenapi_records = util.list_VDI_records_in_sr(sr)

536 for vdi in list(self.__xenapi_records.keys()): 536 ↛ 537line 536 didn't jump to line 537, because the loop on line 536 never started

537 self.__xenapi_locations[util.to_plain_string(self.__xenapi_records[vdi]['location'])] = vdi

538 self.__sm_records = {}

539 for vdi in list(sr.vdis.values()):

540 # We initialise the sm_config field with the values from the database

541 # The sm_config_overrides contains any new fields we want to add to

542 # sm_config, and also any field to delete (by virtue of having

543 # sm_config_overrides[key]=None)

544 try:

545 if not hasattr(vdi, "sm_config"): 545 ↛ 551line 545 didn't jump to line 551, because the condition on line 545 was never false

546 vdi.sm_config = self.__xenapi_records[self.__xenapi_locations[vdi.location]]['sm_config'].copy()

547 except:

548 util.SMlog("missing config for vdi: %s" % vdi.location)

549 vdi.sm_config = {}

550

551 vdi._override_sm_config(vdi.sm_config)

552

553 self.__sm_records[vdi.location] = vdi

554

555 xenapi_locations = set(self.__xenapi_locations.keys())

556 sm_locations = set(self.__sm_records.keys())

557

558 # These ones are new on disk

559 self.new = sm_locations.difference(xenapi_locations)

560 # These have disappeared from the disk

561 self.gone = xenapi_locations.difference(sm_locations)

562 # These are the ones which are still present but might have changed...

563 existing = sm_locations.intersection(xenapi_locations)

564 # Synchronise the uuid fields using the location as the primary key

565 # This ensures we know what the UUIDs are even though they aren't stored

566 # in the storage backend.

567 for location in existing: 567 ↛ 568line 567 didn't jump to line 568, because the loop on line 567 never started

568 sm_vdi = self.get_sm_vdi(location)

569 xenapi_vdi = self.get_xenapi_vdi(location)

570 sm_vdi.uuid = util.default(sm_vdi, "uuid", lambda: xenapi_vdi['uuid'])

571

572 # Only consider those whose configuration looks different

573 self.existing = [x for x in existing if not(self.get_sm_vdi(x).in_sync_with_xenapi_record(self.get_xenapi_vdi(x)))]

574

575 if len(self.new) != 0:

576 util.SMlog("new VDIs on disk: " + repr(self.new))

577 if len(self.gone) != 0: 577 ↛ 578line 577 didn't jump to line 578, because the condition on line 577 was never true

578 util.SMlog("VDIs missing from disk: " + repr(self.gone))

579 if len(self.existing) != 0: 579 ↛ 580line 579 didn't jump to line 580, because the condition on line 579 was never true

580 util.SMlog("VDIs changed on disk: " + repr(self.existing))

581

582 def get_sm_vdi(self, location):

583 return self.__sm_records[location]

584

585 def get_xenapi_vdi(self, location):

586 return self.__xenapi_records[self.__xenapi_locations[location]]

587

588 def all_xenapi_locations(self):

589 return set(self.__xenapi_locations.keys())

590

591 def synchronise_new(self):

592 """Add XenAPI records for new disks"""

593 for location in self.new:

594 vdi = self.get_sm_vdi(location)

595 util.SMlog("Introducing VDI with location=%s" % (vdi.location))

596 vdi._db_introduce()

597

598 def synchronise_gone(self):

599 """Delete XenAPI record for old disks"""

600 for location in self.gone: 600 ↛ 601line 600 didn't jump to line 601, because the loop on line 600 never started

601 vdi = self.get_xenapi_vdi(location)

602 util.SMlog("Forgetting VDI with location=%s uuid=%s" % (util.to_plain_string(vdi['location']), vdi['uuid']))

603 try:

604 self.sr.forget_vdi(vdi['uuid'])

605 except XenAPI.Failure as e:

606 if util.isInvalidVDI(e):

607 util.SMlog("VDI %s not found, ignoring exception" %

608 vdi['uuid'])

609 else:

610 raise

611

612 def synchronise_existing(self):

613 """Update existing XenAPI records"""

614 for location in self.existing: 614 ↛ 615line 614 didn't jump to line 615, because the loop on line 614 never started

615 vdi = self.get_sm_vdi(location)

616

617 util.SMlog("Updating VDI with location=%s uuid=%s" % (vdi.location, vdi.uuid))

618 vdi._db_update()

619

620 def synchronise(self):

621 """Perform the default SM -> xenapi synchronisation; ought to be good enough

622 for most plugins."""

623 self.synchronise_new()

624 self.synchronise_gone()

625 self.synchronise_existing()