Package CedarBackup2 :: Package tools :: Module span
[hide private]
[frames] | no frames]

Source Code for Module CedarBackup2.tools.span

  1  # -*- coding: iso-8859-1 -*- 
  2  # vim: set ft=python ts=3 sw=3 expandtab: 
  3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  4  # 
  5  #              C E D A R 
  6  #          S O L U T I O N S       "Software done right." 
  7  #           S O F T W A R E 
  8  # 
  9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 10  # 
 11  # Copyright (c) 2007-2008,2010 Kenneth J. Pronovici. 
 12  # All rights reserved. 
 13  # 
 14  # This program is free software; you can redistribute it and/or 
 15  # modify it under the terms of the GNU General Public License, 
 16  # Version 2, as published by the Free Software Foundation. 
 17  # 
 18  # This program is distributed in the hope that it will be useful, 
 19  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
 21  # 
 22  # Copies of the GNU General Public License are available from 
 23  # the Free Software Foundation website, http://www.gnu.org/. 
 24  # 
 25  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 26  # 
 27  # Author   : Kenneth J. Pronovici <pronovic@ieee.org> 
 28  # Language : Python (>= 2.5) 
 29  # Project  : Cedar Backup, release 2 
 30  # Revision : $Id: span.py 999 2010-07-07 19:58:25Z pronovic $ 
 31  # Purpose  : Spans staged data among multiple discs 
 32  # 
 33  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 34   
 35  ######################################################################## 
 36  # Notes 
 37  ######################################################################## 
 38   
 39  """ 
 40  Spans staged data among multiple discs 
 41   
 42  This is the Cedar Backup span tool.  It is intended for use by people who stage 
 43  more data than can fit on a single disc.  It allows a user to split staged data 
 44  among more than one disc.  It can't be an extension because it requires user 
 45  input when switching media. 
 46   
 47  Most configuration is taken from the Cedar Backup configuration file, 
 48  specifically the store section.  A few pieces of configuration are taken 
 49  directly from the user. 
 50   
 51  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
 52  """ 
 53   
 54  ######################################################################## 
 55  # Imported modules and constants 
 56  ######################################################################## 
 57   
 58  # System modules 
 59  import sys 
 60  import os 
 61  import logging 
 62  import tempfile 
 63   
 64  # Cedar Backup modules  
 65  from CedarBackup2.release import AUTHOR, EMAIL, VERSION, DATE, COPYRIGHT 
 66  from CedarBackup2.util import displayBytes, convertSize, mount, unmount 
 67  from CedarBackup2.util import UNIT_SECTORS, UNIT_BYTES 
 68  from CedarBackup2.config import Config 
 69  from CedarBackup2.filesystem import BackupFileList, compareDigestMaps, normalizeDir 
 70  from CedarBackup2.cli import Options, setupLogging, setupPathResolver 
 71  from CedarBackup2.cli import DEFAULT_CONFIG, DEFAULT_LOGFILE, DEFAULT_OWNERSHIP, DEFAULT_MODE 
 72  from CedarBackup2.actions.constants import STORE_INDICATOR 
 73  from CedarBackup2.actions.util import createWriter 
 74  from CedarBackup2.actions.store import writeIndicatorFile 
 75  from CedarBackup2.actions.util import findDailyDirs 
 76   
 77   
 78  ######################################################################## 
 79  # Module-wide constants and variables 
 80  ######################################################################## 
 81   
 82  logger = logging.getLogger("CedarBackup2.log.tools.span") 
 83   
 84   
 85  ####################################################################### 
 86  # SpanOptions class 
 87  ####################################################################### 
 88   
89 -class SpanOptions(Options):
90 91 """ 92 Tool-specific command-line options. 93 94 Most of the cback command-line options are exactly what we need here -- 95 logfile path, permissions, verbosity, etc. However, we need to make a few 96 tweaks since we don't accept any actions. 97 98 Also, a few extra command line options that we accept are really ignored 99 underneath. I just don't care about that for a tool like this. 100 """ 101
102 - def validate(self):
103 """ 104 Validates command-line options represented by the object. 105 There are no validations here, because we don't use any actions. 106 @raise ValueError: If one of the validations fails. 107 """ 108 pass
109 110 111 ####################################################################### 112 # Public functions 113 ####################################################################### 114 115 ################# 116 # cli() function 117 ################# 118
119 -def cli():
120 """ 121 Implements the command-line interface for the C{cback-span} script. 122 123 Essentially, this is the "main routine" for the cback-span script. It does 124 all of the argument processing for the script, and then also implements the 125 tool functionality. 126 127 This function looks pretty similiar to C{CedarBackup2.cli.cli()}. It's not 128 easy to refactor this code to make it reusable and also readable, so I've 129 decided to just live with the duplication. 130 131 A different error code is returned for each type of failure: 132 133 - C{1}: The Python interpreter version is < 2.5 134 - C{2}: Error processing command-line arguments 135 - C{3}: Error configuring logging 136 - C{4}: Error parsing indicated configuration file 137 - C{5}: Backup was interrupted with a CTRL-C or similar 138 - C{6}: Error executing other parts of the script 139 140 @note: This script uses print rather than logging to the INFO level, because 141 it is interactive. Underlying Cedar Backup functionality uses the logging 142 mechanism exclusively. 143 144 @return: Error code as described above. 145 """ 146 try: 147 if map(int, [sys.version_info[0], sys.version_info[1]]) < [2, 5]: 148 sys.stderr.write("Python version 2.5 or greater required.\n") 149 return 1 150 except: 151 # sys.version_info isn't available before 2.0 152 sys.stderr.write("Python version 2.5 or greater required.\n") 153 return 1 154 155 try: 156 options = SpanOptions(argumentList=sys.argv[1:]) 157 except Exception, e: 158 _usage() 159 sys.stderr.write(" *** Error: %s\n" % e) 160 return 2 161 162 if options.help: 163 _usage() 164 return 0 165 if options.version: 166 _version() 167 return 0 168 169 try: 170 logfile = setupLogging(options) 171 except Exception, e: 172 sys.stderr.write("Error setting up logging: %s\n" % e) 173 return 3 174 175 logger.info("Cedar Backup 'span' utility run started.") 176 logger.info("Options were [%s]" % options) 177 logger.info("Logfile is [%s]" % logfile) 178 179 if options.config is None: 180 logger.debug("Using default configuration file.") 181 configPath = DEFAULT_CONFIG 182 else: 183 logger.debug("Using user-supplied configuration file.") 184 configPath = options.config 185 186 try: 187 logger.info("Configuration path is [%s]" % configPath) 188 config = Config(xmlPath=configPath) 189 setupPathResolver(config) 190 except Exception, e: 191 logger.error("Error reading or handling configuration: %s" % e) 192 logger.info("Cedar Backup 'span' utility run completed with status 4.") 193 return 4 194 195 if options.stacktrace: 196 _executeAction(options, config) 197 else: 198 try: 199 _executeAction(options, config) 200 except KeyboardInterrupt: 201 logger.error("Backup interrupted.") 202 logger.info("Cedar Backup 'span' utility run completed with status 5.") 203 return 5 204 except Exception, e: 205 logger.error("Error executing backup: %s" % e) 206 logger.info("Cedar Backup 'span' utility run completed with status 6.") 207 return 6 208 209 logger.info("Cedar Backup 'span' utility run completed with status 0.") 210 return 0
211 212 213 ####################################################################### 214 # Utility functions 215 ####################################################################### 216 217 #################### 218 # _usage() function 219 #################### 220
221 -def _usage(fd=sys.stderr):
222 """ 223 Prints usage information for the cback script. 224 @param fd: File descriptor used to print information. 225 @note: The C{fd} is used rather than C{print} to facilitate unit testing. 226 """ 227 fd.write("\n") 228 fd.write(" Usage: cback-span [switches]\n") 229 fd.write("\n") 230 fd.write(" Cedar Backup 'span' tool.\n") 231 fd.write("\n") 232 fd.write(" This Cedar Backup utility spans staged data between multiple discs.\n") 233 fd.write(" It is a utility, not an extension, and requires user interaction.\n") 234 fd.write("\n") 235 fd.write(" The following switches are accepted, mostly to set up underlying\n") 236 fd.write(" Cedar Backup functionality:\n") 237 fd.write("\n") 238 fd.write(" -h, --help Display this usage/help listing\n") 239 fd.write(" -V, --version Display version information\n") 240 fd.write(" -b, --verbose Print verbose output as well as logging to disk\n") 241 fd.write(" -c, --config Path to config file (default: %s)\n" % DEFAULT_CONFIG) 242 fd.write(" -l, --logfile Path to logfile (default: %s)\n" % DEFAULT_LOGFILE) 243 fd.write(" -o, --owner Logfile ownership, user:group (default: %s:%s)\n" % (DEFAULT_OWNERSHIP[0], DEFAULT_OWNERSHIP[1])) 244 fd.write(" -m, --mode Octal logfile permissions mode (default: %o)\n" % DEFAULT_MODE) 245 fd.write(" -O, --output Record some sub-command (i.e. tar) output to the log\n") 246 fd.write(" -d, --debug Write debugging information to the log (implies --output)\n") 247 fd.write(" -s, --stack Dump a Python stack trace instead of swallowing exceptions\n") 248 fd.write("\n")
249 250 251 ###################### 252 # _version() function 253 ###################### 254
255 -def _version(fd=sys.stdout):
256 """ 257 Prints version information for the cback script. 258 @param fd: File descriptor used to print information. 259 @note: The C{fd} is used rather than C{print} to facilitate unit testing. 260 """ 261 fd.write("\n") 262 fd.write(" Cedar Backup 'span' tool.\n") 263 fd.write(" Included with Cedar Backup version %s, released %s.\n" % (VERSION, DATE)) 264 fd.write("\n") 265 fd.write(" Copyright (c) %s %s <%s>.\n" % (COPYRIGHT, AUTHOR, EMAIL)) 266 fd.write(" See CREDITS for a list of included code and other contributors.\n") 267 fd.write(" This is free software; there is NO warranty. See the\n") 268 fd.write(" GNU General Public License version 2 for copying conditions.\n") 269 fd.write("\n") 270 fd.write(" Use the --help option for usage information.\n") 271 fd.write("\n")
272 273 274 ############################ 275 # _executeAction() function 276 ############################ 277
278 -def _executeAction(options, config):
279 """ 280 Implements the guts of the cback-span tool. 281 282 @param options: Program command-line options. 283 @type options: SpanOptions object. 284 285 @param config: Program configuration. 286 @type config: Config object. 287 288 @raise Exception: Under many generic error conditions 289 """ 290 print "" 291 print "================================================" 292 print " Cedar Backup 'span' tool" 293 print "================================================" 294 print "" 295 print "This the Cedar Backup span tool. It is used to split up staging" 296 print "data when that staging data does not fit onto a single disc." 297 print "" 298 print "This utility operates using Cedar Backup configuration. Configuration" 299 print "specifies which staging directory to look at and which writer device" 300 print "and media type to use." 301 print "" 302 if not _getYesNoAnswer("Continue?", default="Y"): 303 return 304 print "===" 305 306 print "" 307 print "Cedar Backup store configuration looks like this:" 308 print "" 309 print " Source Directory...: %s" % config.store.sourceDir 310 print " Media Type.........: %s" % config.store.mediaType 311 print " Device Type........: %s" % config.store.deviceType 312 print " Device Path........: %s" % config.store.devicePath 313 print " Device SCSI ID.....: %s" % config.store.deviceScsiId 314 print " Drive Speed........: %s" % config.store.driveSpeed 315 print " Check Data Flag....: %s" % config.store.checkData 316 print " No Eject Flag......: %s" % config.store.noEject 317 print "" 318 if not _getYesNoAnswer("Is this OK?", default="Y"): 319 return 320 print "===" 321 322 (writer, mediaCapacity) = _getWriter(config) 323 324 print "" 325 print "Please wait, indexing the source directory (this may take a while)..." 326 (dailyDirs, fileList) = _findDailyDirs(config.store.sourceDir) 327 print "===" 328 329 print "" 330 print "The following daily staging directories have not yet been written to disc:" 331 print "" 332 for dailyDir in dailyDirs: 333 print " %s" % dailyDir 334 335 totalSize = fileList.totalSize() 336 print "" 337 print "The total size of the data in these directories is %s." % displayBytes(totalSize) 338 print "" 339 if not _getYesNoAnswer("Continue?", default="Y"): 340 return 341 print "===" 342 343 print "" 344 print "Based on configuration, the capacity of your media is %s." % displayBytes(mediaCapacity) 345 346 print "" 347 print "Since estimates are not perfect and there is some uncertainly in" 348 print "media capacity calculations, it is good to have a \"cushion\"," 349 print "a percentage of capacity to set aside. The cushion reduces the" 350 print "capacity of your media, so a 1.5% cushion leaves 98.5% remaining." 351 print "" 352 cushion = _getFloat("What cushion percentage?", default=4.5) 353 print "===" 354 355 realCapacity = ((100.0 - cushion)/100.0) * mediaCapacity 356 minimumDiscs = (totalSize/realCapacity) + 1 357 print "" 358 print "The real capacity, taking into account the %.2f%% cushion, is %s." % (cushion, displayBytes(realCapacity)) 359 print "It will take at least %d disc(s) to store your %s of data." % (minimumDiscs, displayBytes(totalSize)) 360 print "" 361 if not _getYesNoAnswer("Continue?", default="Y"): 362 return 363 print "===" 364 365 happy = False 366 while not happy: 367 print "" 368 print "Which algorithm do you want to use to span your data across" 369 print "multiple discs?" 370 print "" 371 print "The following algorithms are available:" 372 print "" 373 print " first....: The \"first-fit\" algorithm" 374 print " best.....: The \"best-fit\" algorithm" 375 print " worst....: The \"worst-fit\" algorithm" 376 print " alternate: The \"alternate-fit\" algorithm" 377 print "" 378 print "If you don't like the results you will have a chance to try a" 379 print "different one later." 380 print "" 381 algorithm = _getChoiceAnswer("Which algorithm?", "worst", [ "first", "best", "worst", "alternate", ]) 382 print "===" 383 384 print "" 385 print "Please wait, generating file lists (this may take a while)..." 386 spanSet = fileList.generateSpan(capacity=realCapacity, algorithm="%s_fit" % algorithm) 387 print "===" 388 389 print "" 390 print "Using the \"%s-fit\" algorithm, Cedar Backup can split your data" % algorithm 391 print "into %d discs." % len(spanSet) 392 print "" 393 counter = 0 394 for item in spanSet: 395 counter += 1 396 print "Disc %d: %d files, %s, %.2f%% utilization" % (counter, len(item.fileList), 397 displayBytes(item.size), item.utilization) 398 print "" 399 if _getYesNoAnswer("Accept this solution?", default="Y"): 400 happy = True 401 print "===" 402 403 counter = 0 404 for spanItem in spanSet: 405 counter += 1 406 if counter == 1: 407 print "" 408 _getReturn("Please place the first disc in your backup device.\nPress return when ready.") 409 print "===" 410 else: 411 print "" 412 _getReturn("Please replace the disc in your backup device.\nPress return when ready.") 413 print "===" 414 _writeDisc(config, writer, spanItem) 415 416 _writeStoreIndicator(config, dailyDirs) 417 418 print "" 419 print "Completed writing all discs."
420 421 422 ############################ 423 # _findDailyDirs() function 424 ############################ 425
426 -def _findDailyDirs(stagingDir):
427 """ 428 Returns a list of all daily staging directories that have not yet been 429 stored. 430 431 The store indicator file C{cback.store} will be written to a daily staging 432 directory once that directory is written to disc. So, this function looks 433 at each daily staging directory within the configured staging directory, and 434 returns a list of those which do not contain the indicator file. 435 436 Returned is a tuple containing two items: a list of daily staging 437 directories, and a BackupFileList containing all files among those staging 438 directories. 439 440 @param stagingDir: Configured staging directory 441 442 @return: Tuple (staging dirs, backup file list) 443 """ 444 results = findDailyDirs(stagingDir, STORE_INDICATOR) 445 fileList = BackupFileList() 446 for item in results: 447 fileList.addDirContents(item) 448 return (results, fileList)
449 450 451 ################################## 452 # _writeStoreIndicator() function 453 ################################## 454
455 -def _writeStoreIndicator(config, dailyDirs):
456 """ 457 Writes a store indicator file into daily directories. 458 459 @param config: Config object. 460 @param dailyDirs: List of daily directories 461 """ 462 for dailyDir in dailyDirs: 463 writeIndicatorFile(dailyDir, STORE_INDICATOR, 464 config.options.backupUser, 465 config.options.backupGroup)
466 467 468 ######################## 469 # _getWriter() function 470 ######################## 471
472 -def _getWriter(config):
473 """ 474 Gets a writer and media capacity from store configuration. 475 Returned is a writer and a media capacity in bytes. 476 @param config: Cedar Backup configuration 477 @return: Tuple of (writer, mediaCapacity) 478 """ 479 writer = createWriter(config) 480 mediaCapacity = convertSize(writer.media.capacity, UNIT_SECTORS, UNIT_BYTES) 481 return (writer, mediaCapacity)
482 483 484 ######################## 485 # _writeDisc() function 486 ######################## 487
488 -def _writeDisc(config, writer, spanItem):
489 """ 490 Writes a span item to disc. 491 @param config: Cedar Backup configuration 492 @param writer: Writer to use 493 @param spanItem: Span item to write 494 """ 495 print "" 496 _discInitializeImage(config, writer, spanItem) 497 _discWriteImage(config, writer) 498 _discConsistencyCheck(config, writer, spanItem) 499 print "Write process is complete." 500 print "==="
501
502 -def _discInitializeImage(config, writer, spanItem):
503 """ 504 Initialize an ISO image for a span item. 505 @param config: Cedar Backup configuration 506 @param writer: Writer to use 507 @param spanItem: Span item to write 508 """ 509 complete = False 510 while not complete: 511 try: 512 print "Initializing image..." 513 writer.initializeImage(newDisc=True, tmpdir=config.options.workingDir) 514 for path in spanItem.fileList: 515 graftPoint = os.path.dirname(path.replace(config.store.sourceDir, "", 1)) 516 writer.addImageEntry(path, graftPoint) 517 complete = True 518 except KeyboardInterrupt, e: 519 raise e 520 except Exception, e: 521 logger.error("Failed to initialize image: %s" % e) 522 if not _getYesNoAnswer("Retry initialization step?", default="Y"): 523 raise e 524 print "Ok, attempting retry." 525 print "===" 526 print "Completed initializing image."
527
528 -def _discWriteImage(config, writer):
529 """ 530 Writes a ISO image for a span item. 531 @param config: Cedar Backup configuration 532 @param writer: Writer to use 533 """ 534 complete = False 535 while not complete: 536 try: 537 print "Writing image to disc..." 538 writer.writeImage() 539 complete = True 540 except KeyboardInterrupt, e: 541 raise e 542 except Exception, e: 543 logger.error("Failed to write image: %s" % e) 544 if not _getYesNoAnswer("Retry this step?", default="Y"): 545 raise e 546 print "Ok, attempting retry." 547 _getReturn("Please replace media if needed.\nPress return when ready.") 548 print "===" 549 print "Completed writing image."
550
551 -def _discConsistencyCheck(config, writer, spanItem):
552 """ 553 Run a consistency check on an ISO image for a span item. 554 @param config: Cedar Backup configuration 555 @param writer: Writer to use 556 @param spanItem: Span item to write 557 """ 558 if config.store.checkData: 559 complete = False 560 while not complete: 561 try: 562 print "Running consistency check..." 563 _consistencyCheck(config, spanItem.fileList) 564 complete = True 565 except KeyboardInterrupt, e: 566 raise e 567 except Exception, e: 568 logger.error("Consistency check failed: %s" % e) 569 if not _getYesNoAnswer("Retry the consistency check?", default="Y"): 570 raise e 571 if _getYesNoAnswer("Rewrite the disc first?", default="N"): 572 print "Ok, attempting retry." 573 _getReturn("Please replace the disc in your backup device.\nPress return when ready.") 574 print "===" 575 _discWriteImage(config, writer) 576 else: 577 print "Ok, attempting retry." 578 print "===" 579 print "Completed consistency check."
580 581 582 ############################### 583 # _consistencyCheck() function 584 ############################### 585
586 -def _consistencyCheck(config, fileList):
587 """ 588 Runs a consistency check against media in the backup device. 589 590 The function mounts the device at a temporary mount point in the working 591 directory, and then compares the passed-in file list's digest map with the 592 one generated from the disc. The two lists should be identical. 593 594 If no exceptions are thrown, there were no problems with the consistency 595 check. 596 597 @warning: The implementation of this function is very UNIX-specific. 598 599 @param config: Config object. 600 @param fileList: BackupFileList whose contents to check against 601 602 @raise ValueError: If the check fails 603 @raise IOError: If there is a problem working with the media. 604 """ 605 logger.debug("Running consistency check.") 606 mountPoint = tempfile.mkdtemp(dir=config.options.workingDir) 607 try: 608 mount(config.store.devicePath, mountPoint, "iso9660") 609 discList = BackupFileList() 610 discList.addDirContents(mountPoint) 611 sourceList = BackupFileList() 612 sourceList.extend(fileList) 613 discListDigest = discList.generateDigestMap(stripPrefix=normalizeDir(mountPoint)) 614 sourceListDigest = sourceList.generateDigestMap(stripPrefix=normalizeDir(config.store.sourceDir)) 615 compareDigestMaps(sourceListDigest, discListDigest, verbose=True) 616 logger.info("Consistency check completed. No problems found.") 617 finally: 618 unmount(mountPoint, True, 5, 1) # try 5 times, and remove mount point when done
619 620 621 ######################################################################### 622 # User interface utilities 623 ######################################################################## 624
625 -def _getYesNoAnswer(prompt, default):
626 """ 627 Get a yes/no answer from the user. 628 The default will be placed at the end of the prompt. 629 A "Y" or "y" is considered yes, anything else no. 630 A blank (empty) response results in the default. 631 @param prompt: Prompt to show. 632 @param default: Default to set if the result is blank 633 @return: Boolean true/false corresponding to Y/N 634 """ 635 if default == "Y": 636 prompt = "%s [Y/n]: " % prompt 637 else: 638 prompt = "%s [y/N]: " % prompt 639 answer = raw_input(prompt) 640 if answer in [ None, "", ]: 641 answer = default 642 if answer[0] in [ "Y", "y", ]: 643 return True 644 else: 645 return False
646
647 -def _getChoiceAnswer(prompt, default, validChoices):
648 """ 649 Get a particular choice from the user. 650 The default will be placed at the end of the prompt. 651 The function loops until getting a valid choice. 652 A blank (empty) response results in the default. 653 @param prompt: Prompt to show. 654 @param default: Default to set if the result is None or blank. 655 @param validChoices: List of valid choices (strings) 656 @return: Valid choice from user. 657 """ 658 prompt = "%s [%s]: " % (prompt, default) 659 answer = raw_input(prompt) 660 if answer in [ None, "", ]: 661 answer = default 662 while answer not in validChoices: 663 print "Choice must be one of %s" % validChoices 664 answer = raw_input(prompt) 665 return answer
666
667 -def _getFloat(prompt, default):
668 """ 669 Get a floating point number from the user. 670 The default will be placed at the end of the prompt. 671 The function loops until getting a valid floating point number. 672 A blank (empty) response results in the default. 673 @param prompt: Prompt to show. 674 @param default: Default to set if the result is None or blank. 675 @return: Floating point number from user 676 """ 677 prompt = "%s [%.2f]: " % (prompt, default) 678 while True: 679 answer = raw_input(prompt) 680 if answer in [ None, "" ]: 681 return default 682 else: 683 try: 684 return float(answer) 685 except ValueError: 686 print "Enter a floating point number."
687
688 -def _getReturn(prompt):
689 """ 690 Get a return key from the user. 691 @param prompt: Prompt to show. 692 """ 693 raw_input(prompt)
694 695 696 ######################################################################### 697 # Main routine 698 ######################################################################## 699 700 if __name__ == "__main__": 701 result = cli() 702 sys.exit(result) 703