Coverage for libs/sm/vhdutil.py: 44%

1# Copyright (C) Citrix Systems Inc. 

2# 

3# This program is free software; you can redistribute it and/or modify 

4# it under the terms of the GNU Lesser General Public License as published 

5# by the Free Software Foundation; version 2.1 only. 

6# 

7# This program is distributed in the hope that it will be useful, 

8# but WITHOUT ANY WARRANTY; without even the implied warranty of 

9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

10# GNU Lesser General Public License for more details. 

11# 

12# You should have received a copy of the GNU Lesser General Public License 

13# along with this program; if not, write to the Free Software Foundation, Inc., 

14# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 

15# 

16# Helper functions pertaining to VHD operations 

17# 

18 

19import os 

20from sm.core import util 

21import errno 

22import zlib 

23import re 

24from sm.core import xs_errors 

25import time 

26 

27MIN_VHD_SIZE = 2 * 1024 * 1024 

28MAX_VHD_SIZE = 2040 * 1024 * 1024 * 1024 

29MAX_VHD_JOURNAL_SIZE = 6 * 1024 * 1024 # 2MB VHD block size, max 2TB VHD size 

30MAX_CHAIN_SIZE = 30 # max VHD parent chain size 

31VHD_UTIL = "/usr/bin/vhd-util" 

32OPT_LOG_ERR = "--debug" 

33VHD_BLOCK_SIZE = 2 * 1024 * 1024 

34VHD_FOOTER_SIZE = 512 

35 

36# lock to lock the entire SR for short ops 

37LOCK_TYPE_SR = "sr" 

38 

39VDI_TYPE_VHD = 'vhd' 

40VDI_TYPE_RAW = 'aio' 

41 

42FILE_EXTN_VHD = ".vhd" 

43FILE_EXTN_RAW = ".raw" 

44FILE_EXTN = { 

45 VDI_TYPE_VHD: FILE_EXTN_VHD, 

46 VDI_TYPE_RAW: FILE_EXTN_RAW 

47} 

48 

49 

50class VHDInfo: 

51 uuid = "" 

52 path = "" 

53 sizeVirt = -1 

54 sizePhys = -1 

55 sizeAllocated = -1 

56 hidden = False 

57 parentUuid = "" 

58 parentPath = "" 

59 error = 0 

60 

61 def __init__(self, uuid): 

62 self.uuid = uuid 

63 

64 

65def calcOverheadEmpty(virtual_size): 

66 """Calculate the VHD space overhead (metadata size) for an empty VDI of 

67 size virtual_size""" 

68 overhead = 0 

69 size_mb = virtual_size // (1024 * 1024) 

70 

71 # Footer + footer copy + header + possible CoW parent locator fields 

72 overhead = 3 * 1024 

73 

74 # BAT 4 Bytes per block segment 

75 overhead += (size_mb // 2) * 4 

76 overhead = util.roundup(512, overhead) 

77 

78 # BATMAP 1 bit per block segment 

79 overhead += (size_mb // 2) // 8 

80 overhead = util.roundup(4096, overhead) 

81 

82 return overhead 

83 

84 

85def calcOverheadBitmap(virtual_size): 

86 num_blocks = virtual_size // VHD_BLOCK_SIZE 

87 if virtual_size % VHD_BLOCK_SIZE: 

88 num_blocks += 1 

89 return num_blocks * 4096 

90 

91 

92def ioretry(cmd, text=True): 

93 return util.ioretry(lambda: util.pread2(cmd, text=text), 

94 errlist=[errno.EIO, errno.EAGAIN]) 

95 

96 

97def convertAllocatedSizeToBytes(size): 

98 # Assume we have standard 2MB allocation blocks 

99 return size * 2 * 1024 * 1024 

100 

101 

102def getVHDInfo(path, extractUuidFunction, includeParent=True): 

103 """Get the VHD info. The parent info may optionally be omitted: vhd-util 

104 tries to verify the parent by opening it, which results in error if the VHD 

105 resides on an inactive LV""" 

106 opts = "-vsaf" 

107 if includeParent: 107 ↛ 109line 107 didn't jump to line 109, because the condition on line 107 was never false

108 opts += "p" 

109 cmd = [VHD_UTIL, "query", OPT_LOG_ERR, opts, "-n", path] 

110 ret = ioretry(cmd) 

111 fields = ret.strip().split('\n') 

112 uuid = extractUuidFunction(path) 

113 vhdInfo = VHDInfo(uuid) 

114 vhdInfo.sizeVirt = int(fields[0]) * 1024 * 1024 

115 vhdInfo.sizePhys = int(fields[1]) 

116 nextIndex = 2 

117 if includeParent: 117 ↛ 122line 117 didn't jump to line 122, because the condition on line 117 was never false

118 if fields[nextIndex].find("no parent") == -1: 118 ↛ 119line 118 didn't jump to line 119, because the condition on line 118 was never true

119 vhdInfo.parentPath = fields[nextIndex] 

120 vhdInfo.parentUuid = extractUuidFunction(fields[nextIndex]) 

121 nextIndex += 1 

122 vhdInfo.hidden = int(fields[nextIndex].replace("hidden: ", "")) 

123 vhdInfo.sizeAllocated = convertAllocatedSizeToBytes(int(fields[nextIndex+1])) 

124 vhdInfo.path = path 

125 return vhdInfo 

126 

127 

128def getVHDInfoLVM(lvName, extractUuidFunction, vgName): 

129 """Get the VHD info. This function does not require the container LV to be 

130 active, but uses lvs & vgs""" 

131 vhdInfo = None 

132 cmd = [VHD_UTIL, "scan", "-f", "-l", vgName, "-m", lvName] 

133 ret = ioretry(cmd) 

134 return _parseVHDInfo(ret, extractUuidFunction) 

135 

136 

137def getAllVHDs(pattern, extractUuidFunction, vgName=None, \ 

138 parentsOnly=False, exitOnError=False): 

139 vhds = dict() 

140 cmd = [VHD_UTIL, "scan", "-f", "-m", pattern] 

141 if vgName: 

142 cmd.append("-l") 

143 cmd.append(vgName) 

144 if parentsOnly: 

145 cmd.append("-a") 

146 try: 

147 ret = ioretry(cmd) 

148 except Exception as e: 

149 util.SMlog("WARN: vhd scan failed: output: %s" % e) 

150 ret = ioretry(cmd + ["-c"]) 

151 util.SMlog("WARN: vhd scan with NOFAIL flag, output: %s" % ret) 

152 for line in ret.split('\n'): 

153 if len(line.strip()) == 0: 

154 continue 

155 vhdInfo = _parseVHDInfo(line, extractUuidFunction) 

156 if vhdInfo: 

157 if vhdInfo.error != 0 and exitOnError: 

158 # Just return an empty dict() so the scan will be done 

159 # again by getParentChain. See CA-177063 for details on 

160 # how this has been discovered during the stress tests. 

161 return dict() 

162 vhds[vhdInfo.uuid] = vhdInfo 

163 else: 

164 util.SMlog("WARN: vhdinfo line doesn't parse correctly: %s" % line) 

165 return vhds 

166 

167 

168def getParentChain(lvName, extractUuidFunction, vgName): 

169 """Get the chain of all VHD parents of 'path'. Safe to call for raw VDI's 

170 as well""" 

171 chain = dict() 

172 vdis = dict() 

173 retries = 0 

174 while (not vdis): 

175 if retries > 60: 

176 util.SMlog('ERROR: getAllVHDs returned 0 VDIs after %d retries' % retries) 

177 util.SMlog('ERROR: the VHD metadata might be corrupted') 

178 break 

179 vdis = getAllVHDs(lvName, extractUuidFunction, vgName, True, True) 

180 if (not vdis): 

181 retries = retries + 1 

182 time.sleep(1) 

183 for uuid, vdi in vdis.items(): 

184 chain[uuid] = vdi.path 

185 #util.SMlog("Parent chain for %s: %s" % (lvName, chain)) 

186 return chain 

187 

188 

189def getParent(path, extractUuidFunction): 

190 cmd = [VHD_UTIL, "query", OPT_LOG_ERR, "-p", "-n", path] 

191 ret = ioretry(cmd) 

192 if ret.find("query failed") != -1 or ret.find("Failed opening") != -1: 

193 raise util.SMException("VHD query returned %s" % ret) 

194 if ret.find("no parent") != -1: 

195 return None 

196 return extractUuidFunction(ret) 

197 

198 

199def hasParent(path): 

200 """Check if the VHD has a parent. A VHD has a parent iff its type is 

201 'Differencing'. This function does not need the parent to actually 

202 be present (e.g. the parent LV to be activated).""" 

203 cmd = [VHD_UTIL, "read", OPT_LOG_ERR, "-p", "-n", path] 

204 ret = ioretry(cmd) 

205 # pylint: disable=no-member 

206 m = re.match(".*Disk type\s+: (\S+) hard disk.*", ret, flags=re.S) 

207 vhd_type = m.group(1) 

208 assert(vhd_type == "Differencing" or vhd_type == "Dynamic") 

209 return vhd_type == "Differencing" 

210 

211 

212def setParent(path, parentPath, parentRaw): 

213 normpath = os.path.normpath(parentPath) 

214 cmd = [VHD_UTIL, "modify", OPT_LOG_ERR, "-p", normpath, "-n", path] 

215 if parentRaw: 

216 cmd.append("-m") 

217 ioretry(cmd) 

218 

219 

220def getHidden(path): 

221 cmd = [VHD_UTIL, "query", OPT_LOG_ERR, "-f", "-n", path] 

222 ret = ioretry(cmd) 

223 hidden = int(ret.split(':')[-1].strip()) 

224 return hidden 

225 

226 

227def setHidden(path, hidden=True): 

228 opt = "1" 

229 if not hidden: 229 ↛ 230line 229 didn't jump to line 230, because the condition on line 229 was never true

230 opt = "0" 

231 cmd = [VHD_UTIL, "set", OPT_LOG_ERR, "-n", path, "-f", "hidden", "-v", opt] 

232 ret = ioretry(cmd) 

233 

234 

235def getSizeVirt(path): 

236 cmd = [VHD_UTIL, "query", OPT_LOG_ERR, "-v", "-n", path] 

237 ret = ioretry(cmd) 

238 size = int(ret) * 1024 * 1024 

239 return size 

240 

241 

242def setSizeVirt(path, size, jFile): 

243 "resize VHD offline" 

244 size_mb = size // (1024 * 1024) 

245 cmd = [VHD_UTIL, "resize", OPT_LOG_ERR, "-s", str(size_mb), "-n", path, 

246 "-j", jFile] 

247 ioretry(cmd) 

248 

249 

250def setSizeVirtFast(path, size): 

251 "resize VHD online" 

252 size_mb = size // (1024 * 1024) 

253 cmd = [VHD_UTIL, "resize", OPT_LOG_ERR, "-s", str(size_mb), "-n", path, "-f"] 

254 ioretry(cmd) 

255 

256 

257def getMaxResizeSize(path): 

258 """get the max virtual size for fast resize""" 

259 cmd = [VHD_UTIL, "query", OPT_LOG_ERR, "-S", "-n", path] 

260 ret = ioretry(cmd) 

261 return int(ret) 

262 

263 

264def getSizePhys(path): 

265 cmd = [VHD_UTIL, "query", OPT_LOG_ERR, "-s", "-n", path] 

266 ret = ioretry(cmd) 

267 return int(ret) 

268 

269 

270def setSizePhys(path, size, debug=True): 

271 "set physical utilisation (applicable to VHD's on fixed-size files)" 

272 if debug: 

273 cmd = [VHD_UTIL, "modify", OPT_LOG_ERR, "-s", str(size), "-n", path] 

274 else: 

275 cmd = [VHD_UTIL, "modify", "-s", str(size), "-n", path] 

276 ioretry(cmd) 

277 

278 

279def getAllocatedSize(path): 

280 cmd = [VHD_UTIL, "query", OPT_LOG_ERR, '-a', '-n', path] 

281 ret = ioretry(cmd) 

282 return convertAllocatedSizeToBytes(int(ret)) 

283 

284def killData(path): 

285 "zero out the disk (kill all data inside the VHD file)" 

286 cmd = [VHD_UTIL, "modify", OPT_LOG_ERR, "-z", "-n", path] 

287 ioretry(cmd) 

288 

289 

290def getDepth(path): 

291 "get the VHD parent chain depth" 

292 cmd = [VHD_UTIL, "query", OPT_LOG_ERR, "-d", "-n", path] 

293 text = ioretry(cmd) 

294 depth = -1 

295 if text.startswith("chain depth:"): 

296 depth = int(text.split(':')[1].strip()) 

297 return depth 

298 

299 

300def getBlockBitmap(path): 

301 cmd = [VHD_UTIL, "read", OPT_LOG_ERR, "-B", "-n", path] 

302 text = ioretry(cmd, text=False) 

303 return zlib.compress(text) 

304 

305 

306def coalesce(path): 

307 """ 

308 Coalesce the VHD, on success it returns the number of sectors coalesced 

309 """ 

310 cmd = [VHD_UTIL, "coalesce", OPT_LOG_ERR, "-n", path] 

311 text = ioretry(cmd) 

312 match = re.match(r'^Coalesced (\d+) sectors', text) 

313 if match: 

314 return int(match.group(1)) 

315 

316 return 0 

317 

318 

319def create(path, size, static, msize=0): 

320 size_mb = size // (1024 * 1024) 

321 cmd = [VHD_UTIL, "create", OPT_LOG_ERR, "-n", path, "-s", str(size_mb)] 

322 if static: 

323 cmd.append("-r") 

324 if msize: 

325 cmd.append("-S") 

326 cmd.append(str(msize)) 

327 ioretry(cmd) 

328 

329 

330def snapshot(path, parent, parentRaw, msize=0, checkEmpty=True): 

331 cmd = [VHD_UTIL, "snapshot", OPT_LOG_ERR, "-n", path, "-p", parent] 

332 if parentRaw: 

333 cmd.append("-m") 

334 if msize: 

335 cmd.append("-S") 

336 cmd.append(str(msize)) 

337 if not checkEmpty: 

338 cmd.append("-e") 

339 ioretry(cmd) 

340 

341 

342def check(path, ignoreMissingFooter=False, fast=False): 

343 cmd = [VHD_UTIL, "check", OPT_LOG_ERR, "-n", path] 

344 if ignoreMissingFooter: 

345 cmd.append("-i") 

346 if fast: 

347 cmd.append("-B") 

348 try: 

349 ioretry(cmd) 

350 return True 

351 except util.CommandException: 

352 return False 

353 

354 

355def revert(path, jFile): 

356 cmd = [VHD_UTIL, "revert", OPT_LOG_ERR, "-n", path, "-j", jFile] 

357 ioretry(cmd) 

358 

359 

360def _parseVHDInfo(line, extractUuidFunction): 

361 vhdInfo = None 

362 valueMap = line.split() 

363 if len(valueMap) < 1 or valueMap[0].find("vhd=") == -1: 

364 return None 

365 for keyval in valueMap: 

366 (key, val) = keyval.split('=') 

367 if key == "vhd": 

368 uuid = extractUuidFunction(val) 

369 if not uuid: 

370 util.SMlog("***** malformed output, no UUID: %s" % valueMap) 

371 return None 

372 vhdInfo = VHDInfo(uuid) 

373 vhdInfo.path = val 

374 elif key == "scan-error": 

375 vhdInfo.error = line 

376 util.SMlog("***** VHD scan error: %s" % line) 

377 break 

378 elif key == "capacity": 

379 vhdInfo.sizeVirt = int(val) 

380 elif key == "size": 

381 vhdInfo.sizePhys = int(val) 

382 elif key == "hidden": 

383 vhdInfo.hidden = int(val) 

384 elif key == "parent" and val != "none": 

385 vhdInfo.parentPath = val 

386 vhdInfo.parentUuid = extractUuidFunction(val) 

387 return vhdInfo 

388 

389 

390def _getVHDParentNoCheck(path): 

391 cmd = ["vhd-util", "read", "-p", "-n", "%s" % path] 

392 text = util.pread(cmd) 

393 util.SMlog(text) 

394 for line in text.split('\n'): 

395 if line.find("decoded name :") != -1: 

396 val = line.split(':')[1].strip() 

397 vdi = val.replace("--", "-")[-40:] 

398 if vdi[1:].startswith("LV-"): 

399 vdi = vdi[1:] 

400 return vdi 

401 return None 

402 

403 

404def repair(path): 

405 """Repairs the VHD.""" 

406 ioretry([VHD_UTIL, 'repair', '-n', path]) 

407 

408 

409def validate_and_round_vhd_size(size): 

410 """ Take the supplied vhd size, in bytes, and check it is positive and less 

411 that the maximum supported size, rounding up to the next block boundary 

412 """ 

413 if size < 0 or size > MAX_VHD_SIZE: 

414 raise xs_errors.XenError( 

415 'VDISize', opterr='VDI size ' + 

416 'must be between 1 MB and %d MB' % 

417 (MAX_VHD_SIZE // (1024 * 1024))) 

418 

419 if size < MIN_VHD_SIZE: 419 ↛ 420line 419 didn't jump to line 420, because the condition on line 419 was never true

420 size = MIN_VHD_SIZE 

421 

422 size = util.roundup(VHD_BLOCK_SIZE, size) 

423 

424 return size 

425 

426 

427def getKeyHash(path): 

428 """Extract the hash of the encryption key from the header of an encrypted VHD""" 

429 cmd = ["vhd-util", "key", "-p", "-n", path] 

430 ret = ioretry(cmd) 

431 ret = ret.strip() 

432 if ret == 'none': 

433 return None 

434 vals = ret.split() 

435 if len(vals) != 2: 

436 util.SMlog('***** malformed output from vhd-util' 

437 ' for VHD {}: "{}"'.format(path, ret)) 

438 return None 

439 [_nonce, key_hash] = vals 

440 return key_hash 

441 

442 

443def setKey(path, key_hash): 

444 """Set the encryption key for a VHD""" 

445 cmd = ["vhd-util", "key", "-s", "-n", path, "-H", key_hash] 

446 ioretry(cmd)