Coverage for libs/sm/core/util.py: 47%
1255 statements
« prev ^ index » next coverage.py v7.3.2, created at 2026-07-10 07:59 +0000
1# Copyright (C) Citrix Systems Inc.
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU Lesser General Public License as published
5# by the Free Software Foundation; version 2.1 only.
6#
7# This program is distributed in the hope that it will be useful,
8# but WITHOUT ANY WARRANTY; without even the implied warranty of
9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10# GNU Lesser General Public License for more details.
11#
12# You should have received a copy of the GNU Lesser General Public License
13# along with this program; if not, write to the Free Software Foundation, Inc.,
14# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
15#
16# Miscellaneous utility functions
17#
19import os
20import re
21import sys
22import subprocess
23import shutil
24import tempfile
25import signal
26import time
27import datetime
28import errno
29import socket
30import xml.dom.minidom
31from sm.core import scsiutil
32import stat
33from sm.core import xs_errors
34from sm.core import f_exceptions
35import XenAPI # pylint: disable=import-error
36import xmlrpc.client
37import base64
38import syslog
39import resource
40import traceback
41import glob
42import copy
43import tempfile
45from functools import reduce
47NO_LOGGING_STAMPFILE = '/etc/xensource/no_sm_log'
49IORETRY_MAX = 20 # retries
50IORETRY_PERIOD = 1.0 # seconds
52LOGGING = not (os.path.exists(NO_LOGGING_STAMPFILE))
53_SM_SYSLOG_FACILITY = syslog.LOG_LOCAL2
54LOG_EMERG = syslog.LOG_EMERG
55LOG_ALERT = syslog.LOG_ALERT
56LOG_CRIT = syslog.LOG_CRIT
57LOG_ERR = syslog.LOG_ERR
58LOG_WARNING = syslog.LOG_WARNING
59LOG_NOTICE = syslog.LOG_NOTICE
60LOG_INFO = syslog.LOG_INFO
61LOG_DEBUG = syslog.LOG_DEBUG
63ISCSI_REFDIR = '/run/sr-ref'
65CMD_DD = "/bin/dd"
66CMD_KICKPIPE = '/usr/libexec/sm/kickpipe'
68FIST_PAUSE_PERIOD = 30 # seconds
71class SMException(Exception):
72 """Base class for all SM exceptions for easier catching & wrapping in
73 XenError"""
76class CommandException(SMException):
77 def error_message(self, code):
78 if code > 0:
79 return os.strerror(code)
80 elif code < 0:
81 return "Signalled %s" % (abs(code))
82 return "Success"
84 def __init__(self, code, cmd="", reason='exec failed'):
85 self.code = code
86 self.cmd = cmd
87 self.reason = reason
88 Exception.__init__(self, self.error_message(code))
91class SRBusyException(SMException):
92 """The SR could not be locked"""
93 pass
96def logException(tag):
97 info = sys.exc_info()
98 if info[0] == SystemExit: 98 ↛ 100line 98 didn't jump to line 100, because the condition on line 98 was never true
99 # this should not be happening when catching "Exception", but it is
100 sys.exit(0)
101 tb = reduce(lambda a, b: "%s%s" % (a, b), traceback.format_tb(info[2]))
102 str = "***** %s: EXCEPTION %s, %s\n%s" % (tag, info[0], info[1], tb)
103 SMlog(str)
106def roundup(divisor, value):
107 """Retruns the rounded up value so it is divisible by divisor."""
109 if value == 0: 109 ↛ 110line 109 didn't jump to line 110, because the condition on line 109 was never true
110 value = 1
111 if value % divisor != 0:
112 return ((int(value) // divisor) + 1) * divisor
113 return value
116def to_plain_string(obj):
117 if obj is None:
118 return None
119 if isinstance(obj, dict) and len(obj) == 0: 119 ↛ 120line 119 didn't jump to line 120, because the condition on line 119 was never true
120 SMlog(f"util.to_plain_string() corrected empty dict to empty str")
121 return ""
122 return str(obj)
125def shellquote(arg):
126 return '"%s"' % arg.replace('"', '\\"')
129def make_WWN(name):
130 hex_prefix = name.find("0x")
131 if (hex_prefix >= 0): 131 ↛ 134line 131 didn't jump to line 134, because the condition on line 131 was never false
132 name = name[name.find("0x") + 2:len(name)]
133 # inject dashes for each nibble
134 if (len(name) == 16): # sanity check 134 ↛ 138line 134 didn't jump to line 138, because the condition on line 134 was never false
135 name = name[0:2] + "-" + name[2:4] + "-" + name[4:6] + "-" + \
136 name[6:8] + "-" + name[8:10] + "-" + name[10:12] + "-" + \
137 name[12:14] + "-" + name[14:16]
138 return name
141def _logToSyslog(ident, facility, priority, message):
142 syslog.openlog(ident, 0, facility)
143 syslog.syslog(priority, "[%d] %s" % (os.getpid(), message))
144 syslog.closelog()
147def SMlog(message, ident="SM", priority=LOG_INFO):
148 if LOGGING: 148 ↛ exitline 148 didn't return from function 'SMlog', because the condition on line 148 was never false
149 for message_line in str(message).split('\n'):
150 _logToSyslog(ident, _SM_SYSLOG_FACILITY, priority, message_line)
153def _getDateString():
154 d = datetime.datetime.now()
155 t = d.timetuple()
156 return "%s-%s-%s:%s:%s:%s" % \
157 (t[0], t[1], t[2], t[3], t[4], t[5])
160def doexec(args, inputtext=None, new_env=None, text=True):
161 """Execute a subprocess, then return its return code, stdout and stderr"""
162 env = None
163 if new_env:
164 env = dict(os.environ)
165 env.update(new_env)
166 proc = subprocess.Popen(args, stdin=subprocess.PIPE,
167 stdout=subprocess.PIPE,
168 stderr=subprocess.PIPE,
169 close_fds=True, env=env,
170 universal_newlines=text)
172 if not text and inputtext is not None: 172 ↛ 173line 172 didn't jump to line 173, because the condition on line 172 was never true
173 inputtext = inputtext.encode()
175 (stdout, stderr) = proc.communicate(inputtext)
177 rc = proc.returncode
178 return rc, stdout, stderr
181def is_string(value):
182 return isinstance(value, str)
185# These are partially tested functions that replicate the behaviour of
186# the original pread,pread2 and pread3 functions. Potentially these can
187# replace the original ones at some later date.
188#
189# cmdlist is a list of either single strings or pairs of strings. For
190# each pair, the first component is passed to exec while the second is
191# written to the logs.
192def pread(cmdlist, close_stdin=False, scramble=None, expect_rc=0,
193 quiet=False, new_env=None, text=True):
194 cmdlist_for_exec = []
195 cmdlist_for_log = []
196 for item in cmdlist:
197 if is_string(item): 197 ↛ 207line 197 didn't jump to line 207, because the condition on line 197 was never false
198 cmdlist_for_exec.append(item)
199 if scramble: 199 ↛ 200line 199 didn't jump to line 200, because the condition on line 199 was never true
200 if item.find(scramble) != -1:
201 cmdlist_for_log.append("<filtered out>")
202 else:
203 cmdlist_for_log.append(item)
204 else:
205 cmdlist_for_log.append(item)
206 else:
207 cmdlist_for_exec.append(item[0])
208 cmdlist_for_log.append(item[1])
210 if not quiet: 210 ↛ 212line 210 didn't jump to line 212, because the condition on line 210 was never false
211 SMlog(cmdlist_for_log)
212 (rc, stdout, stderr) = doexec(cmdlist_for_exec, new_env=new_env, text=text)
213 if rc != expect_rc:
214 SMlog("FAILED in util.pread: (rc %d) stdout: '%s', stderr: '%s'" % \
215 (rc, stdout, stderr))
216 if quiet: 216 ↛ 217line 216 didn't jump to line 217, because the condition on line 216 was never true
217 SMlog("Command was: %s" % cmdlist_for_log)
218 if '' == stderr: 218 ↛ 219line 218 didn't jump to line 219, because the condition on line 218 was never true
219 stderr = stdout
220 raise CommandException(rc, str(cmdlist), stderr.strip())
221 if not quiet: 221 ↛ 223line 221 didn't jump to line 223, because the condition on line 221 was never false
222 SMlog(" pread SUCCESS")
223 return stdout
226# POSIX guaranteed atomic within the same file system.
227# Supply directory to ensure tempfile is created
228# in the same directory.
229def atomicFileWrite(targetFile, directory, text):
231 file = None
232 try:
233 # Create file only current pid can write/read to
234 # our responsibility to clean it up.
235 _, tempPath = tempfile.mkstemp(dir=directory)
236 file = open(tempPath, 'w')
237 file.write(text)
239 # Ensure flushed to disk.
240 file.flush()
241 os.fsync(file.fileno())
242 file.close()
244 os.rename(tempPath, targetFile)
245 except OSError:
246 SMlog("FAILED to atomic write to %s" % (targetFile))
248 finally:
249 if (file is not None) and (not file.closed):
250 file.close()
252 if os.path.isfile(tempPath):
253 os.remove(tempPath)
256#Read STDOUT from cmdlist and discard STDERR output
257def pread2(cmdlist, quiet=False, text=True):
258 return pread(cmdlist, quiet=quiet, text=text)
261#Read STDOUT from cmdlist, feeding 'text' to STDIN
262def pread3(cmdlist, text):
263 SMlog(cmdlist)
264 (rc, stdout, stderr) = doexec(cmdlist, text)
265 if rc:
266 SMlog("FAILED in util.pread3: (errno %d) stdout: '%s', stderr: '%s'" % \
267 (rc, stdout, stderr))
268 if '' == stderr:
269 stderr = stdout
270 raise CommandException(rc, str(cmdlist), stderr.strip())
271 SMlog(" pread3 SUCCESS")
272 return stdout
275def listdir(path, quiet=False):
276 cmd = ["ls", path, "-1", "--color=never"]
277 try:
278 text = pread2(cmd, quiet=quiet)[:-1]
279 if len(text) == 0:
280 return []
281 return text.split('\n')
282 except CommandException as inst:
283 if inst.code == errno.ENOENT:
284 raise CommandException(errno.EIO, inst.cmd, inst.reason)
285 else:
286 raise CommandException(inst.code, inst.cmd, inst.reason)
289def gen_uuid():
290 cmd = ["uuidgen", "-r"]
291 return pread(cmd)[:-1]
294def match_uuid(s):
295 regex = re.compile("^[0-9a-f]{8}-(([0-9a-f]{4})-){3}[0-9a-f]{12}")
296 return regex.search(s, 0)
299def findall_uuid(s):
300 regex = re.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
301 return regex.findall(s, 0)
304def exactmatch_uuid(s):
305 regex = re.compile("^[0-9a-f]{8}-(([0-9a-f]{4})-){3}[0-9a-f]{12}$")
306 return regex.search(s, 0)
309def start_log_entry(srpath, path, args):
310 logstring = str(datetime.datetime.now())
311 logstring += " log: "
312 logstring += srpath
313 logstring += " " + path
314 for element in args:
315 logstring += " " + element
316 try:
317 file = open(srpath + "/filelog.txt", "a")
318 file.write(logstring)
319 file.write("\n")
320 file.close()
321 except:
322 pass
324 # failed to write log ...
326def end_log_entry(srpath, path, args):
327 # for teminating, use "error" or "done"
328 logstring = str(datetime.datetime.now())
329 logstring += " end: "
330 logstring += srpath
331 logstring += " " + path
332 for element in args:
333 logstring += " " + element
334 try:
335 file = open(srpath + "/filelog.txt", "a")
336 file.write(logstring)
337 file.write("\n")
338 file.close()
339 except:
340 pass
342 # failed to write log ...
343 # for now print
344 # print "%s" % logstring
346def ioretry(f, errlist=[errno.EIO], maxretry=IORETRY_MAX, period=IORETRY_PERIOD, **ignored):
347 retries = 0
348 while True:
349 try:
350 return f()
351 except OSError as ose:
352 err = int(ose.errno)
353 if not err in errlist:
354 raise CommandException(err, str(f), "OSError")
355 except CommandException as ce:
356 if not int(ce.code) in errlist:
357 raise
359 retries += 1
360 if retries >= maxretry:
361 break
363 time.sleep(period)
365 raise CommandException(errno.ETIMEDOUT, str(f), "Timeout")
368def ioretry_stat(path, maxretry=IORETRY_MAX):
369 # this ioretry is similar to the previous method, but
370 # stat does not raise an error -- so check its return
371 retries = 0
372 while retries < maxretry:
373 stat = os.statvfs(path)
374 if stat.f_blocks != -1:
375 return stat
376 time.sleep(1)
377 retries += 1
378 raise CommandException(errno.EIO, "os.statvfs")
381def sr_get_capability(sr_uuid, session=None):
382 result = []
383 local_session = None
384 if session is None:
385 local_session = get_localAPI_session()
386 session = local_session
388 try:
389 sr_ref = session.xenapi.SR.get_by_uuid(sr_uuid)
390 sm_type = session.xenapi.SR.get_record(sr_ref)['type']
391 sm_rec = session.xenapi.SM.get_all_records_where(
392 "field \"type\" = \"%s\"" % sm_type)
394 # SM expects at least one entry of any SR type
395 if len(sm_rec) > 0:
396 result = list(sm_rec.values())[0]['capabilities']
398 return result
399 finally:
400 if local_session:
401 local_session.xenapi.session.logout()
403def sr_get_driver_info(driver_info):
404 results = {}
405 # first add in the vanilla stuff
406 for key in ['name', 'description', 'vendor', 'copyright', \
407 'driver_version', 'required_api_version']:
408 results[key] = driver_info[key]
409 # add the capabilities (xmlrpc array)
410 # enforcing activate/deactivate for blktap2
411 caps = driver_info['capabilities']
412 if "ATOMIC_PAUSE" in caps: 412 ↛ 413line 412 didn't jump to line 413, because the condition on line 412 was never true
413 for cap in ("VDI_ACTIVATE", "VDI_DEACTIVATE"):
414 if not cap in caps:
415 caps.append(cap)
416 elif "VDI_ACTIVATE" in caps or "VDI_DEACTIVATE" in caps: 416 ↛ 417line 416 didn't jump to line 417, because the condition on line 416 was never true
417 SMlog("Warning: vdi_[de]activate present for %s" % driver_info["name"])
419 results['capabilities'] = caps
420 # add in the configuration options
421 options = []
422 for option in driver_info['configuration']:
423 options.append({'key': option[0], 'description': option[1]})
424 results['configuration'] = options
425 return xmlrpc.client.dumps((results, ), "", True)
428def return_nil():
429 return xmlrpc.client.dumps((None, ), "", True, allow_none=True)
432def SRtoXML(SRlist):
433 dom = xml.dom.minidom.Document()
434 driver = dom.createElement("SRlist")
435 dom.appendChild(driver)
437 for key in SRlist.keys():
438 dict = SRlist[key]
439 entry = dom.createElement("SR")
440 driver.appendChild(entry)
442 e = dom.createElement("UUID")
443 entry.appendChild(e)
444 textnode = dom.createTextNode(key)
445 e.appendChild(textnode)
447 if 'size' in dict:
448 e = dom.createElement("Size")
449 entry.appendChild(e)
450 textnode = dom.createTextNode(str(dict['size']))
451 e.appendChild(textnode)
453 if 'storagepool' in dict:
454 e = dom.createElement("StoragePool")
455 entry.appendChild(e)
456 textnode = dom.createTextNode(str(dict['storagepool']))
457 e.appendChild(textnode)
459 if 'aggregate' in dict:
460 e = dom.createElement("Aggregate")
461 entry.appendChild(e)
462 textnode = dom.createTextNode(str(dict['aggregate']))
463 e.appendChild(textnode)
465 return dom.toprettyxml()
468def pathexists(path):
469 try:
470 os.lstat(path)
471 return True
472 except OSError as inst:
473 if inst.errno == errno.EIO: 473 ↛ 474line 473 didn't jump to line 474, because the condition on line 473 was never true
474 time.sleep(1)
475 try:
476 listdir(os.path.realpath(os.path.dirname(path)))
477 os.lstat(path)
478 return True
479 except:
480 pass
481 raise CommandException(errno.EIO, "os.lstat(%s)" % path, "failed")
482 return False
485def force_unlink(path):
486 try:
487 os.unlink(path)
488 except OSError as e:
489 if e.errno != errno.ENOENT: 489 ↛ 490line 489 didn't jump to line 490, because the condition on line 489 was never true
490 raise
493def create_secret(session, secret):
494 ref = session.xenapi.secret.create({'value': secret})
495 return session.xenapi.secret.get_uuid(ref)
498def get_secret(session, uuid):
499 try:
500 ref = session.xenapi.secret.get_by_uuid(uuid)
501 return session.xenapi.secret.get_value(ref)
502 except:
503 raise xs_errors.XenError('InvalidSecret', opterr='Unable to look up secret [%s]' % uuid)
506def wait_for_path(path, timeout):
507 for i in range(0, timeout): 507 ↛ 511line 507 didn't jump to line 511, because the loop on line 507 didn't complete
508 if len(glob.glob(path)): 508 ↛ 510line 508 didn't jump to line 510, because the condition on line 508 was never false
509 return True
510 time.sleep(1)
511 return False
514def wait_for_nopath(path, timeout):
515 for i in range(0, timeout): 515 ↛ 519line 515 didn't jump to line 519, because the loop on line 515 didn't complete
516 if not os.path.exists(path): 516 ↛ 518line 516 didn't jump to line 518, because the condition on line 516 was never false
517 return True
518 time.sleep(1)
519 return False
522def wait_for_path_multi(path, timeout):
523 for i in range(0, timeout):
524 paths = glob.glob(path)
525 SMlog("_wait_for_paths_multi: paths = %s" % paths)
526 if len(paths):
527 SMlog("_wait_for_paths_multi: return first path: %s" % paths[0])
528 return paths[0]
529 time.sleep(1)
530 return ""
533def isdir(path):
534 try:
535 st = os.stat(path)
536 return stat.S_ISDIR(st.st_mode)
537 except OSError as inst:
538 if inst.errno == errno.EIO: 538 ↛ 539line 538 didn't jump to line 539, because the condition on line 538 was never true
539 raise CommandException(errno.EIO, "os.stat(%s)" % path, "failed")
540 return False
543def get_single_entry(path):
544 f = open(path, 'r')
545 line = f.readline()
546 f.close()
547 return line.rstrip()
550def get_fs_size(path):
551 st = ioretry_stat(path)
552 return st.f_blocks * st.f_frsize
555def get_fs_utilisation(path):
556 st = ioretry_stat(path)
557 return (st.f_blocks - st.f_bfree) * \
558 st.f_frsize
561def ismount(path):
562 """Test whether a path is a mount point"""
563 try:
564 s1 = os.stat(path)
565 s2 = os.stat(os.path.join(path, '..'))
566 except OSError as inst:
567 raise CommandException(inst.errno, "os.stat")
568 dev1 = s1.st_dev
569 dev2 = s2.st_dev
570 if dev1 != dev2:
571 return True # path/.. on a different device as path
572 ino1 = s1.st_ino
573 ino2 = s2.st_ino
574 if ino1 == ino2:
575 return True # path/.. is the same i-node as path
576 return False
579def makedirs(name, mode=0o777):
580 head, tail = os.path.split(name)
581 if not tail: 581 ↛ 582line 581 didn't jump to line 582, because the condition on line 581 was never true
582 head, tail = os.path.split(head)
583 if head and tail and not pathexists(head):
584 makedirs(head, mode)
585 if tail == os.curdir: 585 ↛ 586line 585 didn't jump to line 586, because the condition on line 585 was never true
586 return
587 try:
588 os.mkdir(name, mode)
589 except OSError as exc:
590 if exc.errno == errno.EEXIST and os.path.isdir(name):
591 if mode:
592 os.chmod(name, mode)
593 pass
594 else:
595 raise
598def zeroOut(path, fromByte, bytes):
599 """write 'bytes' zeros to 'path' starting from fromByte (inclusive)"""
600 blockSize = 4096
602 fromBlock = fromByte // blockSize
603 if fromByte % blockSize:
604 fromBlock += 1
605 bytesBefore = fromBlock * blockSize - fromByte
606 if bytesBefore > bytes:
607 bytesBefore = bytes
608 bytes -= bytesBefore
609 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=1",
610 "seek=%s" % fromByte, "count=%s" % bytesBefore]
611 try:
612 pread2(cmd)
613 except CommandException:
614 return False
616 blocks = bytes // blockSize
617 bytes -= blocks * blockSize
618 fromByte = (fromBlock + blocks) * blockSize
619 if blocks:
620 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=%s" % blockSize,
621 "seek=%s" % fromBlock, "count=%s" % blocks]
622 try:
623 pread2(cmd)
624 except CommandException:
625 return False
627 if bytes:
628 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=1",
629 "seek=%s" % fromByte, "count=%s" % bytes]
630 try:
631 pread2(cmd)
632 except CommandException:
633 return False
635 return True
638def match_rootdev(s):
639 regex = re.compile("^PRIMARY_DISK")
640 return regex.search(s, 0)
643def getrootdev():
644 filename = '/etc/xensource-inventory'
645 try:
646 f = open(filename, 'r')
647 except:
648 raise xs_errors.XenError('EIO', \
649 opterr="Unable to open inventory file [%s]" % filename)
650 rootdev = ''
651 for line in filter(match_rootdev, f.readlines()):
652 rootdev = line.split("'")[1]
653 if not rootdev: 653 ↛ 654line 653 didn't jump to line 654, because the condition on line 653 was never true
654 raise xs_errors.XenError('NoRootDev')
655 return rootdev
658def getrootdevID():
659 rootdev = getrootdev()
660 try:
661 rootdevID = scsiutil.getSCSIid(rootdev)
662 except:
663 SMlog("util.getrootdevID: Unable to verify serial or SCSIid of device: %s" \
664 % rootdev)
665 return ''
667 if not len(rootdevID):
668 SMlog("util.getrootdevID: Unable to identify scsi device [%s] via scsiID" \
669 % rootdev)
671 return rootdevID
674def get_localAPI_session():
675 # First acquire a valid session
676 session = XenAPI.xapi_local()
677 try:
678 session.xenapi.login_with_password('root', '', '', 'SM')
679 except:
680 raise xs_errors.XenError('APISession')
681 return session
684def get_this_host():
685 uuid = None
686 f = open("/etc/xensource-inventory", 'r')
687 for line in f.readlines():
688 if line.startswith("INSTALLATION_UUID"):
689 uuid = line.split("'")[1]
690 f.close()
691 return uuid
694def is_master(session):
695 pools = session.xenapi.pool.get_all()
696 master = session.xenapi.pool.get_master(pools[0])
697 return get_this_host_ref(session) == master
700def get_localhost_ref(session):
701 filename = '/etc/xensource-inventory'
702 try:
703 f = open(filename, 'r')
704 except:
705 raise xs_errors.XenError('EIO', \
706 opterr="Unable to open inventory file [%s]" % filename)
707 domid = ''
708 for line in filter(match_domain_id, f.readlines()):
709 domid = line.split("'")[1]
710 if not domid:
711 raise xs_errors.XenError('APILocalhost')
713 vms = session.xenapi.VM.get_all_records_where('field "uuid" = "%s"' % domid)
714 for vm in vms:
715 record = vms[vm]
716 if record["uuid"] == domid:
717 hostid = record["resident_on"]
718 return hostid
719 raise xs_errors.XenError('APILocalhost')
722def match_domain_id(s):
723 regex = re.compile("^CONTROL_DOMAIN_UUID")
724 return regex.search(s, 0)
727def get_hosts_attached_on(session, vdi_uuids):
728 host_refs = {}
729 for vdi_uuid in vdi_uuids:
730 try:
731 vdi_ref = session.xenapi.VDI.get_by_uuid(vdi_uuid)
732 except XenAPI.Failure:
733 SMlog("VDI %s not in db, ignoring" % vdi_uuid)
734 continue
735 sm_config = session.xenapi.VDI.get_sm_config(vdi_ref)
736 for key in [x for x in sm_config.keys() if x.startswith('host_')]:
737 host_refs[key[len('host_'):]] = True
738 return host_refs.keys()
741def get_this_host_ref(session):
742 host_uuid = get_this_host()
743 host_ref = session.xenapi.host.get_by_uuid(host_uuid)
744 return host_ref
747def get_slaves_attached_on(session, vdi_uuids):
748 "assume this host is the SR master"
749 host_refs = get_hosts_attached_on(session, vdi_uuids)
750 master_ref = get_this_host_ref(session)
751 return [x for x in host_refs if x != master_ref]
754def get_online_hosts(session):
755 online_hosts = []
756 hosts = session.xenapi.host.get_all_records()
757 for host_ref, host_rec in hosts.items():
758 metricsRef = host_rec["metrics"]
759 metrics = session.xenapi.host_metrics.get_record(metricsRef)
760 if metrics["live"]:
761 online_hosts.append(host_ref)
762 return online_hosts
765def get_all_slaves(session):
766 "assume this host is the SR master"
767 host_refs = get_online_hosts(session)
768 master_ref = get_this_host_ref(session)
769 return [x for x in host_refs if x != master_ref]
772def is_attached_rw(sm_config):
773 for key, val in sm_config.items():
774 if key.startswith("host_") and val == "RW":
775 return True
776 return False
779def attached_as(sm_config):
780 for key, val in sm_config.items():
781 if key.startswith("host_") and (val == "RW" or val == "RO"): 781 ↛ 782line 781 didn't jump to line 782, because the condition on line 781 was never true
782 return val
785def find_my_pbd_record(session, host_ref, sr_ref):
786 try:
787 pbds = session.xenapi.PBD.get_all_records()
788 for pbd_ref in pbds.keys():
789 if pbds[pbd_ref]['host'] == host_ref and pbds[pbd_ref]['SR'] == sr_ref:
790 return [pbd_ref, pbds[pbd_ref]]
791 return None
792 except Exception as e:
793 SMlog("Caught exception while looking up PBD for host %s SR %s: %s" % (str(host_ref), str(sr_ref), str(e)))
794 return None
797def find_my_pbd(session, host_ref, sr_ref):
798 ret = find_my_pbd_record(session, host_ref, sr_ref)
799 if ret is not None:
800 return ret[0]
801 else:
802 return None
805def test_hostPBD_devs(session, sr_uuid, devs):
806 host = get_localhost_ref(session)
807 sr = session.xenapi.SR.get_by_uuid(sr_uuid)
808 try:
809 pbds = session.xenapi.PBD.get_all_records()
810 except:
811 raise xs_errors.XenError('APIPBDQuery')
812 for dev in devs.split(','):
813 for pbd in pbds:
814 record = pbds[pbd]
815 # it's ok if it's *our* PBD
816 if record["SR"] == sr:
817 break
818 if record["host"] == host:
819 devconfig = record["device_config"]
820 if 'device' in devconfig:
821 for device in devconfig['device'].split(','):
822 if os.path.realpath(device) == os.path.realpath(dev):
823 return True
824 return False
827def test_hostPBD_lun(session, targetIQN, LUNid):
828 host = get_localhost_ref(session)
829 try:
830 pbds = session.xenapi.PBD.get_all_records()
831 except:
832 raise xs_errors.XenError('APIPBDQuery')
833 for pbd in pbds:
834 record = pbds[pbd]
835 if record["host"] == host:
836 devconfig = record["device_config"]
837 if 'targetIQN' in devconfig and 'LUNid' in devconfig:
838 if devconfig['targetIQN'] == targetIQN and \
839 devconfig['LUNid'] == LUNid:
840 return True
841 return False
844def test_SCSIid(session, sr_uuid, SCSIid):
845 if sr_uuid is not None:
846 sr = session.xenapi.SR.get_by_uuid(sr_uuid)
847 try:
848 pbds = session.xenapi.PBD.get_all_records()
849 except:
850 raise xs_errors.XenError('APIPBDQuery')
851 for pbd in pbds:
852 record = pbds[pbd]
853 # it's ok if it's *our* PBD
854 # During FC SR creation, devscan.py passes sr_uuid as None
855 if sr_uuid is not None:
856 if record["SR"] == sr:
857 break
858 devconfig = record["device_config"]
859 sm_config = session.xenapi.SR.get_sm_config(record["SR"])
860 if 'SCSIid' in devconfig and devconfig['SCSIid'] == SCSIid:
861 return True
862 elif 'SCSIid' in sm_config and sm_config['SCSIid'] == SCSIid:
863 return True
864 elif 'scsi-' + SCSIid in sm_config:
865 return True
866 return False
869class TimeoutException(SMException):
870 pass
873def timeout_call(timeoutseconds, function, *arguments):
874 def handler(signum, frame):
875 raise TimeoutException()
876 signal.signal(signal.SIGALRM, handler)
877 signal.alarm(timeoutseconds)
878 try:
879 return function(*arguments)
880 finally:
881 signal.alarm(0)
884def _incr_iscsiSR_refcount(targetIQN, uuid):
885 if not os.path.exists(ISCSI_REFDIR):
886 os.mkdir(ISCSI_REFDIR)
887 filename = os.path.join(ISCSI_REFDIR, targetIQN)
888 try:
889 f = open(filename, 'a+')
890 except:
891 raise xs_errors.XenError('LVMRefCount', \
892 opterr='file %s' % filename)
894 f.seek(0)
895 found = False
896 refcount = 0
897 for line in filter(match_uuid, f.readlines()):
898 refcount += 1
899 if line.find(uuid) != -1:
900 found = True
901 if not found:
902 f.write("%s\n" % uuid)
903 refcount += 1
904 f.close()
905 return refcount
908def _decr_iscsiSR_refcount(targetIQN, uuid):
909 filename = os.path.join(ISCSI_REFDIR, targetIQN)
910 if not os.path.exists(filename):
911 return 0
912 try:
913 f = open(filename, 'a+')
914 except:
915 raise xs_errors.XenError('LVMRefCount', \
916 opterr='file %s' % filename)
918 f.seek(0)
919 output = []
920 refcount = 0
921 for line in filter(match_uuid, f.readlines()):
922 if line.find(uuid) == -1:
923 output.append(line.rstrip())
924 refcount += 1
925 if not refcount:
926 os.unlink(filename)
927 return refcount
929 # Re-open file and truncate
930 f.close()
931 f = open(filename, 'w')
932 for i in range(0, refcount):
933 f.write("%s\n" % output[i])
934 f.close()
935 return refcount
938# The agent enforces 1 PBD per SR per host, so we
939# check for active SR entries not attached to this host
940def test_activePoolPBDs(session, host, uuid):
941 try:
942 pbds = session.xenapi.PBD.get_all_records()
943 except:
944 raise xs_errors.XenError('APIPBDQuery')
945 for pbd in pbds:
946 record = pbds[pbd]
947 if record["host"] != host and record["SR"] == uuid \
948 and record["currently_attached"]:
949 return True
950 return False
953def remove_mpathcount_field(session, host_ref, sr_ref, SCSIid):
954 try:
955 pbdref = find_my_pbd(session, host_ref, sr_ref)
956 if pbdref is not None:
957 key = "mpath-" + SCSIid
958 session.xenapi.PBD.remove_from_other_config(pbdref, key)
959 except:
960 pass
963def kickpipe_mpathcount():
964 """
965 Issue a kick to the mpathcount service. This will ensure that mpathcount runs
966 shortly to update the multipath config records, if it was not already activated
967 by a UDEV event.
968 """
969 cmd = [CMD_KICKPIPE, "mpathcount"]
970 (rc, stdout, stderr) = doexec(cmd)
971 return (rc == 0)
974def _testHost(hostname, port, errstring, timeout=5):
975 SMlog("_testHost: Testing host/port: %s,%d" % (hostname, port))
976 try:
977 sockinfo = socket.getaddrinfo(hostname, int(port))[0]
978 except:
979 logException('Exception occured getting IP for %s' % hostname)
980 raise xs_errors.XenError('DNSError')
982 sock = socket.socket(sockinfo[0], socket.SOCK_STREAM)
983 # Only allow the connect to block for up to timeout seconds
984 sock.settimeout(timeout)
985 try:
986 sock.connect(sockinfo[4])
987 # Fix for MS storage server bug
988 sock.send(b'\n')
989 sock.close()
990 except socket.error as reason:
991 SMlog("_testHost: Connect failed after %d seconds (%s) - %s" \
992 % (timeout, hostname, reason))
993 raise xs_errors.XenError(errstring)
995def testHost(hostname, port):
996 """
997 Wrapper for _testHost which returns boolean. This also uses the
998 f_exceptions wrapper which serves as a test.
999 """
1000 try:
1001 # The third argument doesn't really matter here because we always
1002 # swallow the exception and return False. It just needs to be a
1003 # valid argument to xs_errors.XenError()
1004 _testHost(hostname, port, "ISCSITarget(SMAPIv3)", timeout=10)
1005 except f_exceptions.XenError:
1006 return False
1007 return True
1009def match_scsiID(s, id):
1010 regex = re.compile(id)
1011 return regex.search(s, 0)
1014def _isSCSIid(s):
1015 regex = re.compile("^scsi-")
1016 return regex.search(s, 0)
1019def test_scsiserial(session, device):
1020 device = os.path.realpath(device)
1021 if not scsiutil._isSCSIdev(device):
1022 SMlog("util.test_scsiserial: Not a serial device: %s" % device)
1023 return False
1024 serial = ""
1025 try:
1026 serial += scsiutil.getserial(device)
1027 except:
1028 # Error allowed, SCSIid is the important one
1029 pass
1031 try:
1032 scsiID = scsiutil.getSCSIid(device)
1033 except:
1034 SMlog("util.test_scsiserial: Unable to verify serial or SCSIid of device: %s" \
1035 % device)
1036 return False
1037 if not len(scsiID):
1038 SMlog("util.test_scsiserial: Unable to identify scsi device [%s] via scsiID" \
1039 % device)
1040 return False
1042 try:
1043 SRs = session.xenapi.SR.get_all_records()
1044 except:
1045 raise xs_errors.XenError('APIFailure')
1046 for SR in SRs:
1047 record = SRs[SR]
1048 conf = record["sm_config"]
1049 if 'devserial' in conf:
1050 for dev in conf['devserial'].split(','):
1051 if _isSCSIid(dev):
1052 if match_scsiID(dev, scsiID):
1053 return True
1054 elif len(serial) and dev == serial:
1055 return True
1056 return False
1059def default(self, field, thunk):
1060 try:
1061 return getattr(self, field)
1062 except:
1063 return thunk()
1066def list_VDI_records_in_sr(sr):
1067 """Helper function which returns a list of all VDI records for this SR
1068 stored in the XenAPI server, useful for implementing SR.scan"""
1069 sr_ref = sr.session.xenapi.SR.get_by_uuid(sr.uuid)
1070 vdis = sr.session.xenapi.VDI.get_all_records_where("field \"SR\" = \"%s\"" % sr_ref)
1071 return vdis
1074# Given a partition (e.g. sda1), get a disk name:
1075def diskFromPartition(partition):
1076 # check whether this is a device mapper device (e.g. /dev/dm-0)
1077 m = re.match('(/dev/)?(dm-[0-9]+)(p[0-9]+)?$', partition)
1078 if m is not None: 1078 ↛ 1079line 1078 didn't jump to line 1079, because the condition on line 1078 was never true
1079 return m.group(2)
1081 numlen = 0 # number of digit characters
1082 m = re.match("\D+(\d+)", partition)
1083 if m is not None: 1083 ↛ 1084line 1083 didn't jump to line 1084, because the condition on line 1083 was never true
1084 numlen = len(m.group(1))
1086 # is it a cciss?
1087 if True in [partition.startswith(x) for x in ['cciss', 'ida', 'rd']]: 1087 ↛ 1088line 1087 didn't jump to line 1088, because the condition on line 1087 was never true
1088 numlen += 1 # need to get rid of trailing 'p'
1090 # is it a mapper path?
1091 if partition.startswith("mapper"): 1091 ↛ 1092line 1091 didn't jump to line 1092, because the condition on line 1091 was never true
1092 if re.search("p[0-9]*$", partition):
1093 numlen = len(re.match("\d+", partition[::-1]).group(0)) + 1
1094 SMlog("Found mapper part, len %d" % numlen)
1095 else:
1096 numlen = 0
1098 # is it /dev/disk/by-id/XYZ-part<k>?
1099 if partition.startswith("disk/by-id"): 1099 ↛ 1100line 1099 didn't jump to line 1100, because the condition on line 1099 was never true
1100 return partition[:partition.rfind("-part")]
1102 return partition[:len(partition) - numlen]
1105def dom0_disks():
1106 """Disks carrying dom0, e.g. ['/dev/sda']"""
1107 disks = []
1108 with open("/etc/mtab", 'r') as f: 1108 ↛ 1109, 1108 ↛ 11152 missed branches: 1) line 1108 didn't jump to line 1109, 2) line 1108 didn't jump to line 1115
1109 for line in f:
1110 (dev, mountpoint, fstype, opts, freq, passno) = line.split(' ')
1111 if mountpoint == '/':
1112 disk = diskFromPartition(dev)
1113 if not (disk in disks):
1114 disks.append(disk)
1115 SMlog("Dom0 disks: %s" % disks)
1116 return disks
1119def set_scheduler_sysfs_node(node, scheds):
1120 """
1121 Set the scheduler for a sysfs node (e.g. '/sys/block/sda')
1122 according to prioritized list schedulers
1123 Try to set the first item, then fall back to the next on failure
1124 """
1126 path = os.path.join(node, "queue", "scheduler")
1127 if not os.path.exists(path): 1127 ↛ 1131line 1127 didn't jump to line 1131, because the condition on line 1127 was never false
1128 SMlog("no path %s" % path)
1129 return
1131 stored_error = None
1132 for sched in scheds:
1133 try:
1134 with open(path, 'w') as file:
1135 file.write("%s\n" % sched)
1136 SMlog("Set scheduler to [%s] on [%s]" % (sched, node))
1137 return
1138 except (OSError, IOError) as err:
1139 stored_error = err
1141 SMlog("Error setting schedulers to [%s] on [%s], %s" % (scheds, node, str(stored_error)))
1144def set_scheduler(dev, schedulers=None):
1145 if schedulers is None: 1145 ↛ 1148line 1145 didn't jump to line 1148, because the condition on line 1145 was never false
1146 schedulers = ["none", "noop"]
1148 devices = []
1149 if not scsiutil.match_dm(dev): 1149 ↛ 1153line 1149 didn't jump to line 1153, because the condition on line 1149 was never false
1150 # Remove partition numbers
1151 devices.append(diskFromPartition(dev).replace('/', '!'))
1152 else:
1153 rawdev = diskFromPartition(dev)
1154 devices = [os.path.realpath(x)[5:] for x in scsiutil._genReverseSCSIidmap(rawdev.split('/')[-1])]
1156 for d in devices:
1157 set_scheduler_sysfs_node("/sys/block/%s" % d, schedulers)
1160# This function queries XAPI for the existing VDI records for this SR
1161def _getVDIs(srobj):
1162 VDIs = []
1163 try:
1164 sr_ref = getattr(srobj, 'sr_ref')
1165 except AttributeError:
1166 return VDIs
1168 refs = srobj.session.xenapi.SR.get_VDIs(sr_ref)
1169 for vdi in refs:
1170 ref = srobj.session.xenapi.VDI.get_record(vdi)
1171 ref['vdi_ref'] = vdi
1172 VDIs.append(ref)
1173 return VDIs
1176def _getVDI(srobj, vdi_uuid):
1177 vdi = srobj.session.xenapi.VDI.get_by_uuid(vdi_uuid)
1178 ref = srobj.session.xenapi.VDI.get_record(vdi)
1179 ref['vdi_ref'] = vdi
1180 return ref
1183def _convertDNS(name):
1184 addr = socket.getaddrinfo(name, None)[0][4][0]
1185 return addr
1188def _containsVDIinuse(srobj):
1189 VDIs = _getVDIs(srobj)
1190 for vdi in VDIs:
1191 if not vdi['managed']:
1192 continue
1193 sm_config = vdi['sm_config']
1194 if 'SRRef' in sm_config:
1195 try:
1196 PBDs = srobj.session.xenapi.SR.get_PBDs(sm_config['SRRef'])
1197 for pbd in PBDs:
1198 record = PBDs[pbd]
1199 if record["host"] == srobj.host_ref and \
1200 record["currently_attached"]:
1201 return True
1202 except:
1203 pass
1204 return False
1207def isVDICommand(cmd):
1208 if cmd is None or cmd in ["vdi_attach", "vdi_detach",
1209 "vdi_activate", "vdi_deactivate",
1210 "vdi_epoch_begin", "vdi_epoch_end"]:
1211 return True
1212 else:
1213 return False
1216#########################
1217# Daemon helper functions
1218def p_id_fork():
1219 try:
1220 p_id = os.fork()
1221 except OSError as e:
1222 print("Fork failed: %s (%d)" % (e.strerror, e.errno))
1223 sys.exit(-1)
1225 if (p_id == 0):
1226 os.setsid()
1227 try:
1228 p_id = os.fork()
1229 except OSError as e:
1230 print("Fork failed: %s (%d)" % (e.strerror, e.errno))
1231 sys.exit(-1)
1232 if (p_id == 0):
1233 os.chdir('/')
1234 os.umask(0)
1235 else:
1236 os._exit(0)
1237 else:
1238 os._exit(0)
1241def daemon():
1242 p_id_fork()
1243 # Query the max file descriptor parameter for this process
1244 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1246 # Close any fds that are open
1247 for fd in range(0, maxfd):
1248 try:
1249 os.close(fd)
1250 except:
1251 pass
1253 # Redirect STDIN to STDOUT and STDERR
1254 os.open('/dev/null', os.O_RDWR)
1255 os.dup2(0, 1)
1256 os.dup2(0, 2)
1258################################################################################
1259#
1260# Fist points
1261#
1263# * The global variable 'fistpoint' define the list of all possible fistpoints;
1264#
1265# * To activate a fistpoint called 'name', you need to create the file '/tmp/fist_name'
1266# on the SR master;
1267#
1268# * At the moment, activating a fist point can lead to two possible behaviors:
1269# - if '/tmp/fist_LVHDRT_exit' exists, then the function called during the fistpoint is _exit;
1270# - otherwise, the function called is _pause.
1272def _pause(secs, name):
1273 SMlog("Executing fist point %s: sleeping %d seconds ..." % (name, secs))
1274 time.sleep(secs)
1275 SMlog("Executing fist point %s: done" % name)
1278def _exit(name):
1279 SMlog("Executing fist point %s: exiting the current process ..." % name)
1280 raise xs_errors.XenError('FistPoint', opterr='%s' % name)
1283class FistPoint:
1284 def __init__(self, points):
1285 #SMlog("Fist points loaded")
1286 self.points = points
1288 def is_legal(self, name):
1289 return (name in self.points)
1291 def is_active(self, name):
1292 return os.path.exists("/tmp/fist_%s" % name)
1294 def mark_sr(self, name, sruuid, started):
1295 session = get_localAPI_session()
1296 try:
1297 sr = session.xenapi.SR.get_by_uuid(sruuid)
1299 if started:
1300 session.xenapi.SR.add_to_other_config(sr, name, "active")
1301 else:
1302 session.xenapi.SR.remove_from_other_config(sr, name)
1303 finally:
1304 session.xenapi.session.logout()
1306 def activate(self, name, sruuid):
1307 if name in self.points:
1308 if self.is_active(name):
1309 self.mark_sr(name, sruuid, True)
1310 if self.is_active("LVHDRT_exit"): 1310 ↛ 1311line 1310 didn't jump to line 1311, because the condition on line 1310 was never true
1311 self.mark_sr(name, sruuid, False)
1312 _exit(name)
1313 else:
1314 _pause(FIST_PAUSE_PERIOD, name)
1315 self.mark_sr(name, sruuid, False)
1316 else:
1317 SMlog("Unknown fist point: %s" % name)
1319 def activate_custom_fn(self, name, fn):
1320 if name in self.points: 1320 ↛ 1326line 1320 didn't jump to line 1326, because the condition on line 1320 was never false
1321 if self.is_active(name): 1321 ↛ 1322line 1321 didn't jump to line 1322, because the condition on line 1321 was never true
1322 SMlog("Executing fist point %s: starting ..." % name)
1323 fn()
1324 SMlog("Executing fist point %s: done" % name)
1325 else:
1326 SMlog("Unknown fist point: %s" % name)
1329def list_find(f, seq):
1330 for item in seq:
1331 if f(item):
1332 return item
1334GCPAUSE_FISTPOINT = "GCLoop_no_pause"
1336fistpoint = FistPoint(["LVHDRT_finding_a_suitable_pair",
1337 "LVHDRT_inflating_the_parent",
1338 "LVHDRT_resizing_while_vdis_are_paused",
1339 "LVHDRT_coalescing_VHD_data",
1340 "LVHDRT_coalescing_before_inflate_grandparent",
1341 "LVHDRT_relinking_grandchildren",
1342 "LVHDRT_before_create_relink_journal",
1343 "LVHDRT_xapiSM_serialization_tests",
1344 "LVHDRT_clone_vdi_after_create_journal",
1345 "LVHDRT_clone_vdi_after_shrink_parent",
1346 "LVHDRT_clone_vdi_after_first_snap",
1347 "LVHDRT_clone_vdi_after_second_snap",
1348 "LVHDRT_clone_vdi_after_parent_hidden",
1349 "LVHDRT_clone_vdi_after_parent_ro",
1350 "LVHDRT_clone_vdi_before_remove_journal",
1351 "LVHDRT_clone_vdi_after_lvcreate",
1352 "LVHDRT_clone_vdi_before_undo_clone",
1353 "LVHDRT_clone_vdi_after_undo_clone",
1354 "LVHDRT_inflate_after_create_journal",
1355 "LVHDRT_inflate_after_setSize",
1356 "LVHDRT_inflate_after_zeroOut",
1357 "LVHDRT_inflate_after_setSizePhys",
1358 "LVHDRT_inflate_after_setSizePhys",
1359 "LVHDRT_coaleaf_before_coalesce",
1360 "LVHDRT_coaleaf_after_coalesce",
1361 "LVHDRT_coaleaf_one_renamed",
1362 "LVHDRT_coaleaf_both_renamed",
1363 "LVHDRT_coaleaf_after_vdirec",
1364 "LVHDRT_coaleaf_before_delete",
1365 "LVHDRT_coaleaf_after_delete",
1366 "LVHDRT_coaleaf_before_remove_j",
1367 "LVHDRT_coaleaf_undo_after_rename",
1368 "LVHDRT_coaleaf_undo_after_rename2",
1369 "LVHDRT_coaleaf_undo_after_refcount",
1370 "LVHDRT_coaleaf_undo_after_deflate",
1371 "LVHDRT_coaleaf_undo_end",
1372 "LVHDRT_coaleaf_stop_after_recovery",
1373 "LVHDRT_coaleaf_finish_after_inflate",
1374 "LVHDRT_coaleaf_finish_end",
1375 "LVHDRT_coaleaf_delay_1",
1376 "LVHDRT_coaleaf_delay_2",
1377 "LVHDRT_coaleaf_delay_3",
1378 "testsm_clone_allow_raw",
1379 "xenrt_default_vdi_type_legacy",
1380 "blktap_activate_inject_failure",
1381 "blktap_activate_error_handling",
1382 GCPAUSE_FISTPOINT,
1383 "cleanup_coalesceVHD_inject_failure",
1384 "cleanup_tracker_no_progress",
1385 "FileSR_fail_hardlink",
1386 "FileSR_fail_snap1",
1387 "FileSR_fail_snap2",
1388 "LVM_journaler_exists",
1389 "LVM_journaler_none",
1390 "LVM_journaler_badname",
1391 "LVM_journaler_readfail",
1392 "LVM_journaler_writefail"])
1395def set_dirty(session, sr):
1396 try:
1397 session.xenapi.SR.add_to_other_config(sr, "dirty", "")
1398 SMlog("set_dirty %s succeeded" % (repr(sr)))
1399 except:
1400 SMlog("set_dirty %s failed (flag already set?)" % (repr(sr)))
1403def doesFileHaveOpenHandles(fileName):
1404 SMlog("Entering doesFileHaveOpenHandles with file: %s" % fileName)
1405 (retVal, processAndPidTuples) = \
1406 findRunningProcessOrOpenFile(fileName, False)
1408 if not retVal:
1409 SMlog("Failed to determine if file %s has open handles." % \
1410 fileName)
1411 # err on the side of caution
1412 return True
1413 else:
1414 if len(processAndPidTuples) > 0:
1415 return True
1416 else:
1417 return False
1420# extract SR uuid from the passed in devmapper entry and return
1421# /dev/mapper/VG_XenStorage--c3d82e92--cb25--c99b--b83a--482eebab4a93-MGT
1422def extractSRFromDevMapper(path):
1423 try:
1424 path = os.path.basename(path)
1425 path = path[len('VG_XenStorage-') + 1:]
1426 path = path.replace('--', '/')
1427 path = path[0:path.rfind('-')]
1428 return path.replace('/', '-')
1429 except:
1430 return ''
1433def pid_is_alive(pid):
1434 """
1435 Try to kill PID with signal 0.
1436 If we succeed, the PID is alive, so return True.
1437 If we get an EPERM error, the PID is alive but we are not allowed to
1438 signal it. Still return true.
1439 Any other error (e.g. ESRCH), return False
1440 """
1441 try:
1442 os.kill(pid, 0)
1443 return True
1444 except OSError as e:
1445 if e.errno == errno.EPERM:
1446 return True
1447 return False
1450# Looks at /proc and figures either
1451# If a process is still running (default), returns open file names
1452# If any running process has open handles to the given file (process = False)
1453# returns process names and pids
1454def findRunningProcessOrOpenFile(name, process=True):
1455 retVal = True
1456 links = []
1457 processandpids = []
1458 sockets = set()
1459 try:
1460 SMlog("Entering findRunningProcessOrOpenFile with params: %s" % \
1461 [name, process])
1463 # Look at all pids
1464 pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]
1465 for pid in sorted(pids):
1466 try:
1467 try:
1468 f = None
1469 f = open(os.path.join('/proc', pid, 'cmdline'), 'r')
1470 prog = f.read()[:-1]
1471 if prog: 1471 ↛ 1480line 1471 didn't jump to line 1480, because the condition on line 1471 was never false
1472 # Just want the process name
1473 argv = prog.split('\x00')
1474 prog = argv[0]
1475 except IOError as e:
1476 if e.errno in (errno.ENOENT, errno.ESRCH):
1477 SMlog("ERROR %s reading %s, ignore" % (e.errno, pid))
1478 continue
1479 finally:
1480 if f is not None: 1480 ↛ 1465, 1480 ↛ 14832 missed branches: 1) line 1480 didn't jump to line 1465, because the continue on line 1478 wasn't executed, 2) line 1480 didn't jump to line 1483, because the condition on line 1480 was never false
1481 f.close() 1481 ↛ 1465line 1481 didn't jump to line 1465, because the continue on line 1478 wasn't executed
1483 try:
1484 fd_dir = os.path.join('/proc', pid, 'fd')
1485 files = os.listdir(fd_dir)
1486 except OSError as e:
1487 if e.errno in (errno.ENOENT, errno.ESRCH):
1488 SMlog("ERROR %s reading fds for %s, ignore" % (e.errno, pid))
1489 # Ignore pid that are no longer valid
1490 continue
1491 else:
1492 raise
1494 for file in files:
1495 try:
1496 link = os.readlink(os.path.join(fd_dir, file))
1497 except OSError:
1498 continue
1500 if process: 1500 ↛ 1505line 1500 didn't jump to line 1505, because the condition on line 1500 was never false
1501 if name == prog: 1501 ↛ 1494line 1501 didn't jump to line 1494, because the condition on line 1501 was never false
1502 links.append(link)
1503 else:
1504 # need to return process name and pid tuples
1505 if link == name:
1506 processandpids.append((prog, pid))
1508 # Get the connected sockets
1509 if name == prog:
1510 sockets.update(get_connected_sockets(pid))
1512 # We will only have a non-empty processandpids if some fd entries were found.
1513 # Before returning them, verify that all the PIDs in question are properly alive.
1514 # There is no specific guarantee of when a PID's /proc directory will disappear
1515 # when it exits, particularly relative to filedescriptor cleanup, so we want to
1516 # make sure we're not reporting a false positive.
1517 processandpids = [x for x in processandpids if pid_is_alive(int(x[1]))]
1518 for pp in processandpids: 1518 ↛ 1519line 1518 didn't jump to line 1519, because the loop on line 1518 never started
1519 SMlog(f"File {name} has an open handle with process {pp[0]} with pid {pp[1]}")
1521 except Exception as e:
1522 SMlog("Exception checking running process or open file handles. " \
1523 "Error: %s" % str(e))
1524 retVal = False
1526 if process: 1526 ↛ 1529line 1526 didn't jump to line 1529, because the condition on line 1526 was never false
1527 return retVal, links, sockets
1528 else:
1529 return retVal, processandpids
1532def get_connected_sockets(pid):
1533 sockets = set()
1534 try:
1535 # Lines in /proc/<pid>/net/unix are formatted as follows
1536 # (see Linux source net/unix/af_unix.c, unix_seq_show() )
1537 # - Pointer address to socket (hex)
1538 # - Refcount (HEX)
1539 # - 0
1540 # - State (HEX, 0 or __SO_ACCEPTCON)
1541 # - Type (HEX - but only 0001 of interest)
1542 # - Connection state (HEX - but only 03, SS_CONNECTED of interest)
1543 # - Inode number
1544 # - Path (optional)
1545 open_sock_matcher = re.compile(
1546 r'^[0-9a-f]+: [0-9A-Fa-f]+ [0-9A-Fa-f]+ [0-9A-Fa-f]+ 0001 03 \d+ (.*)$')
1547 with open(
1548 os.path.join('/proc', str(pid), 'net', 'unix'), 'r') as f:
1549 lines = f.readlines()
1550 for line in lines:
1551 match = open_sock_matcher.match(line)
1552 if match:
1553 sockets.add(match[1])
1554 except OSError as e:
1555 if e.errno in (errno.ENOENT, errno.ESRCH):
1556 # Ignore pid that are no longer valid
1557 SMlog("ERROR %s reading sockets for %s, ignore" %
1558 (e.errno, pid))
1559 else:
1560 raise
1561 return sockets
1564def retry(f, maxretry=20, period=3):
1565 retries = 0
1566 while True:
1567 try:
1568 return f()
1569 except Exception as e:
1570 SMlog("Got exception: %s. Retry number: %s" % (str(e), retries))
1572 retries += 1
1573 if retries >= maxretry:
1574 break
1576 time.sleep(period)
1578 return f()
1581def getCslDevPath(svid):
1582 basepath = "/dev/disk/by-csldev/"
1583 if svid.startswith("NETAPP_"):
1584 # special attention for NETAPP SVIDs
1585 svid_parts = svid.split("__")
1586 globstr = basepath + "NETAPP__LUN__" + "*" + svid_parts[2] + "*" + svid_parts[-1] + "*"
1587 else:
1588 globstr = basepath + svid + "*"
1590 return globstr
1593# Use device in /dev pointed to by cslg path which consists of svid
1594def get_scsiid_from_svid(md_svid):
1595 cslg_path = getCslDevPath(md_svid)
1596 abs_path = glob.glob(cslg_path)
1597 if abs_path:
1598 real_path = os.path.realpath(abs_path[0])
1599 return scsiutil.getSCSIid(real_path)
1600 else:
1601 return None
1604def get_isl_scsiids(session):
1605 # Get cslg type SRs
1606 SRs = session.xenapi.SR.get_all_records_where('field "type" = "cslg"')
1608 # Iterate through the SR to get the scsi ids
1609 scsi_id_ret = []
1610 for SR in SRs:
1611 sr_rec = SRs[SR]
1612 # Use the md_svid to get the scsi id
1613 scsi_id = get_scsiid_from_svid(sr_rec['sm_config']['md_svid'])
1614 if scsi_id:
1615 scsi_id_ret.append(scsi_id)
1617 # Get the vdis in the SR and do the same procedure
1618 vdi_recs = session.xenapi.VDI.get_all_records_where('field "SR" = "%s"' % SR)
1619 for vdi_rec in vdi_recs:
1620 vdi_rec = vdi_recs[vdi_rec]
1621 scsi_id = get_scsiid_from_svid(vdi_rec['sm_config']['SVID'])
1622 if scsi_id:
1623 scsi_id_ret.append(scsi_id)
1625 return scsi_id_ret
1628def get_scsi_id(path):
1629 """
1630 Compatibility wrapper for sm-core-libs which had its own copy
1631 of scsiutil.getSCSIid(). Converts any CommandException raised
1632 to a XenError as consumers of sm-core-libs are expecting.
1633 """
1634 try:
1635 return scsiutil.getSCSIid(path)
1636 except CommandException as e:
1637 raise f_exceptions.XenError("Command", e.reason)
1640class extractXVA:
1641 # streams files as a set of file and checksum, caller should remove
1642 # the files, if not needed. The entire directory (Where the files
1643 # and checksum) will only be deleted as part of class cleanup.
1644 HDR_SIZE = 512
1645 BLOCK_SIZE = 512
1646 SIZE_LEN = 12 - 1 # To remove \0 from tail
1647 SIZE_OFFSET = 124
1648 ZERO_FILLED_REC = 2
1649 NULL_IDEN = '\x00'
1650 DIR_IDEN = '/'
1651 CHECKSUM_IDEN = '.checksum'
1652 OVA_FILE = 'ova.xml'
1654 # Init gunzips the file using a subprocess, and reads stdout later
1655 # as and when needed
1656 def __init__(self, filename):
1657 self.__extract_path = ''
1658 self.__filename = filename
1659 cmd = 'gunzip -cd %s' % filename
1660 try:
1661 self.spawn_p = subprocess.Popen(
1662 cmd, shell=True, \
1663 stdin=subprocess.PIPE, stdout=subprocess.PIPE, \
1664 stderr=subprocess.PIPE, close_fds=True)
1665 except Exception as e:
1666 SMlog("Error: %s. Uncompress failed for %s" % (str(e), filename))
1667 raise Exception(str(e))
1669 # Create dir to extract the files
1670 self.__extract_path = tempfile.mkdtemp()
1672 def __del__(self):
1673 shutil.rmtree(self.__extract_path)
1675 # Class supports Generator expression. 'for f_name, checksum in getTuple()'
1676 # returns filename, checksum content. Returns filename, '' in case
1677 # of checksum file missing. e.g. ova.xml
1678 def getTuple(self):
1679 zerod_record = 0
1680 ret_f_name = ''
1681 ret_base_f_name = ''
1683 try:
1684 # Read tar file as sets of file and checksum.
1685 while True:
1686 # Read the output of spawned process, or output of gunzip
1687 f_hdr = self.spawn_p.stdout.read(self.HDR_SIZE)
1689 # Break out in case of end of file
1690 if f_hdr == '':
1691 if zerod_record == extractXVA.ZERO_FILLED_REC:
1692 break
1693 else:
1694 SMlog('Error. Expects %d zero records', \
1695 extractXVA.ZERO_FILLED_REC)
1696 raise Exception('Unrecognized end of file')
1698 # Watch out for zero records, two zero records
1699 # denote end of file.
1700 if f_hdr == extractXVA.NULL_IDEN * extractXVA.HDR_SIZE:
1701 zerod_record += 1
1702 continue
1704 f_name = f_hdr[:f_hdr.index(extractXVA.NULL_IDEN)]
1705 # File header may be for a folder, if so ignore the header
1706 if not f_name.endswith(extractXVA.DIR_IDEN):
1707 f_size_octal = f_hdr[extractXVA.SIZE_OFFSET: \
1708 extractXVA.SIZE_OFFSET + extractXVA.SIZE_LEN]
1709 f_size = int(f_size_octal, 8)
1710 if f_name.endswith(extractXVA.CHECKSUM_IDEN):
1711 if f_name.rstrip(extractXVA.CHECKSUM_IDEN) == \
1712 ret_base_f_name:
1713 checksum = self.spawn_p.stdout.read(f_size)
1714 yield(ret_f_name, checksum)
1715 else:
1716 # Expects file followed by its checksum
1717 SMlog('Error. Sequence mismatch starting with %s', \
1718 ret_f_name)
1719 raise Exception( \
1720 'Files out of sequence starting with %s', \
1721 ret_f_name)
1722 else:
1723 # In case of ova.xml, read the contents into a file and
1724 # return the file name to the caller. For other files,
1725 # read the contents into a file, it will
1726 # be used when a .checksum file is encountered.
1727 ret_f_name = '%s/%s' % (self.__extract_path, f_name)
1728 ret_base_f_name = f_name
1730 # Check if the folder exists on the target location,
1731 # else create it.
1732 folder_path = ret_f_name[:ret_f_name.rfind('/')]
1733 if not os.path.exists(folder_path):
1734 os.mkdir(folder_path)
1736 # Store the file to the tmp folder, strip the tail \0
1737 f = open(ret_f_name, 'w')
1738 f.write(self.spawn_p.stdout.read(f_size))
1739 f.close()
1740 if f_name == extractXVA.OVA_FILE:
1741 yield(ret_f_name, '')
1743 # Skip zero'd portion of data block
1744 round_off = f_size % extractXVA.BLOCK_SIZE
1745 if round_off != 0:
1746 zeros = self.spawn_p.stdout.read(
1747 extractXVA.BLOCK_SIZE - round_off)
1748 except Exception as e:
1749 SMlog("Error: %s. File set extraction failed %s" % (str(e), \
1750 self.__filename))
1752 # Kill and Drain stdout of the gunzip process,
1753 # else gunzip might block on stdout
1754 os.kill(self.spawn_p.pid, signal.SIGTERM)
1755 self.spawn_p.communicate()
1756 raise Exception(str(e))
1758illegal_xml_chars = [(0x00, 0x08), (0x0B, 0x1F), (0x7F, 0x84), (0x86, 0x9F),
1759 (0xD800, 0xDFFF), (0xFDD0, 0xFDDF), (0xFFFE, 0xFFFF),
1760 (0x1FFFE, 0x1FFFF), (0x2FFFE, 0x2FFFF), (0x3FFFE, 0x3FFFF),
1761 (0x4FFFE, 0x4FFFF), (0x5FFFE, 0x5FFFF), (0x6FFFE, 0x6FFFF),
1762 (0x7FFFE, 0x7FFFF), (0x8FFFE, 0x8FFFF), (0x9FFFE, 0x9FFFF),
1763 (0xAFFFE, 0xAFFFF), (0xBFFFE, 0xBFFFF), (0xCFFFE, 0xCFFFF),
1764 (0xDFFFE, 0xDFFFF), (0xEFFFE, 0xEFFFF), (0xFFFFE, 0xFFFFF),
1765 (0x10FFFE, 0x10FFFF)]
1767illegal_ranges = ["%s-%s" % (chr(low), chr(high))
1768 for (low, high) in illegal_xml_chars
1769 if low < sys.maxunicode]
1771illegal_xml_re = re.compile(u'[%s]' % u''.join(illegal_ranges))
1774def isLegalXMLString(s):
1775 """Tells whether this is a valid XML string (i.e. it does not contain
1776 illegal XML characters specified in
1777 http://www.w3.org/TR/2004/REC-xml-20040204/#charsets).
1778 """
1780 if len(s) > 0:
1781 return re.search(illegal_xml_re, s) is None
1782 else:
1783 return True
1786def unictrunc(string, max_bytes):
1787 """
1788 Given a string, returns the largest number of elements for a prefix
1789 substring of it, such that the UTF-8 encoding of this substring takes no
1790 more than the given number of bytes.
1792 The string may be given as a unicode string or a UTF-8 encoded byte
1793 string, and the number returned will be in characters or bytes
1794 accordingly. Note that in the latter case, the substring will still be a
1795 valid UTF-8 encoded string (which is to say, it won't have been truncated
1796 part way through a multibyte sequence for a unicode character).
1798 string: the string to truncate
1799 max_bytes: the maximum number of bytes the truncated string can be
1800 """
1801 if isinstance(string, str):
1802 return_chars = True
1803 else:
1804 return_chars = False
1805 string = string.decode('UTF-8')
1807 cur_chars = 0
1808 cur_bytes = 0
1809 for char in string:
1810 charsize = len(char.encode('UTF-8'))
1811 if cur_bytes + charsize > max_bytes:
1812 break
1813 else:
1814 cur_chars += 1
1815 cur_bytes += charsize
1816 return cur_chars if return_chars else cur_bytes
1819def hideValuesInPropMap(propmap, propnames):
1820 """
1821 Worker function: input simple map of prop name/value pairs, and
1822 a list of specific propnames whose values we want to hide.
1823 Loop through the "hide" list, and if any are found, hide the
1824 value and return the altered map.
1825 If none found, return the original map
1826 """
1827 matches = []
1828 for propname in propnames:
1829 if propname in propmap: 1829 ↛ 1830line 1829 didn't jump to line 1830, because the condition on line 1829 was never true
1830 matches.append(propname)
1832 if matches: 1832 ↛ 1833line 1832 didn't jump to line 1833, because the condition on line 1832 was never true
1833 deepCopyRec = copy.deepcopy(propmap)
1834 for match in matches:
1835 deepCopyRec[match] = '******'
1836 return deepCopyRec
1838 return propmap
1839# define the list of propnames whose value we want to hide
1841PASSWD_PROP_KEYS = ['password', 'cifspassword', 'chappassword', 'incoming_chappassword']
1842DEFAULT_SEGMENT_LEN = 950
1845def hidePasswdInConfig(config):
1846 """
1847 Function to hide passwd values in a simple prop map,
1848 for example "device_config"
1849 """
1850 return hideValuesInPropMap(config, PASSWD_PROP_KEYS)
1853def hidePasswdInParams(params, configProp):
1854 """
1855 Function to hide password values in a specified property which
1856 is a simple map of prop name/values, and is itself an prop entry
1857 in a larger property map.
1858 For example, param maps containing "device_config", or
1859 "sm_config", etc
1860 """
1861 params[configProp] = hideValuesInPropMap(params[configProp], PASSWD_PROP_KEYS)
1862 return params
1865def hideMemberValuesInXmlParams(xmlParams, propnames=PASSWD_PROP_KEYS):
1866 """
1867 Function to hide password values in XML params, specifically
1868 for the XML format of incoming params to SR modules.
1869 Uses text parsing: loop through the list of specific propnames
1870 whose values we want to hide, and:
1871 - Assemble a full "prefix" containing each property name, e.g.,
1872 "<member><name>password</name><value>"
1873 - Test the XML if it contains that string, save the index.
1874 - If found, get the index of the ending tag
1875 - Truncate the return string starting with the password value.
1876 - Append the substitute "*******" value string.
1877 - Restore the rest of the original string starting with the end tag.
1878 """
1879 findStrPrefixHead = "<member><name>"
1880 findStrPrefixTail = "</name><value>"
1881 findStrSuffix = "</value>"
1882 strlen = len(xmlParams)
1884 for propname in propnames:
1885 findStrPrefix = findStrPrefixHead + propname + findStrPrefixTail
1886 idx = xmlParams.find(findStrPrefix)
1887 if idx != -1: # if found any of them
1888 idx += len(findStrPrefix)
1889 idx2 = xmlParams.find(findStrSuffix, idx)
1890 if idx2 != -1:
1891 retStr = xmlParams[0:idx]
1892 retStr += "******"
1893 retStr += xmlParams[idx2:strlen]
1894 return retStr
1895 else:
1896 return xmlParams
1897 return xmlParams
1900def splitXmlText(xmlData, segmentLen=DEFAULT_SEGMENT_LEN, showContd=False):
1901 """
1902 Split xml string data into substrings small enough for the
1903 syslog line length limit. Split at tag end markers ( ">" ).
1904 Usage:
1905 strList = []
1906 strList = splitXmlText( longXmlText, maxLineLen ) # maxLineLen is optional
1907 """
1908 remainingData = str(xmlData)
1910 # "Un-pretty-print"
1911 remainingData = remainingData.replace('\n', '')
1912 remainingData = remainingData.replace('\t', '')
1914 remainingChars = len(remainingData)
1915 returnData = ''
1917 thisLineNum = 0
1918 while remainingChars > segmentLen:
1919 thisLineNum = thisLineNum + 1
1920 index = segmentLen
1921 tmpStr = remainingData[:segmentLen]
1922 tmpIndex = tmpStr.rfind('>')
1923 if tmpIndex != -1:
1924 index = tmpIndex + 1
1926 tmpStr = tmpStr[:index]
1927 remainingData = remainingData[index:]
1928 remainingChars = len(remainingData)
1930 if showContd:
1931 if thisLineNum != 1:
1932 tmpStr = '(Cont\'d): ' + tmpStr
1933 tmpStr = tmpStr + ' (Cont\'d):'
1935 returnData += tmpStr + '\n'
1937 if showContd and thisLineNum > 0:
1938 remainingData = '(Cont\'d): ' + remainingData
1939 returnData += remainingData
1941 return returnData
1944def inject_failure():
1945 raise Exception('injected failure')
1948def open_atomic(path, mode=None):
1949 """Atomically creates a file if, and only if it does not already exist.
1950 Leaves the file open and returns the file object.
1952 path: the path to atomically open
1953 mode: "r" (read), "w" (write), or "rw" (read/write)
1954 returns: an open file object"""
1956 assert path
1958 flags = os.O_CREAT | os.O_EXCL
1959 modes = {'r': os.O_RDONLY, 'w': os.O_WRONLY, 'rw': os.O_RDWR}
1960 if mode:
1961 if mode not in modes:
1962 raise Exception('invalid access mode ' + mode)
1963 flags |= modes[mode]
1964 fd = os.open(path, flags)
1965 try:
1966 if mode:
1967 return os.fdopen(fd, mode)
1968 else:
1969 return os.fdopen(fd)
1970 except:
1971 os.close(fd)
1972 raise
1975def isInvalidVDI(exception):
1976 return exception.details[0] == "HANDLE_INVALID" or \
1977 exception.details[0] == "UUID_INVALID"
1980def get_pool_restrictions(session):
1981 """Returns pool restrictions as a map, @session must be already
1982 established."""
1983 return list(session.xenapi.pool.get_all_records().values())[0]['restrictions']
1986def read_caching_is_restricted(session):
1987 """Tells whether read caching is restricted."""
1988 if session is None: 1988 ↛ 1989line 1988 didn't jump to line 1989, because the condition on line 1988 was never true
1989 return True
1990 restrictions = get_pool_restrictions(session)
1991 if 'restrict_read_caching' in restrictions and \ 1991 ↛ 1993line 1991 didn't jump to line 1993, because the condition on line 1991 was never true
1992 restrictions['restrict_read_caching'] == "true":
1993 return True
1994 return False
1997def sessions_less_than_targets(other_config, device_config):
1998 if 'multihomelist' in device_config and 'iscsi_sessions' in other_config:
1999 sessions = int(other_config['iscsi_sessions'])
2000 targets = len(device_config['multihomelist'].split(','))
2001 SMlog("Targets %d and iscsi_sessions %d" % (targets, sessions))
2002 return (sessions < targets)
2003 else:
2004 return False