1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 """
39 Spans staged data among multiple discs
40
41 This is the Cedar Backup span tool. It is intended for use by people who stage
42 more data than can fit on a single disc. It allows a user to split staged data
43 among more than one disc. It can't be an extension because it requires user
44 input when switching media.
45
46 Most configuration is taken from the Cedar Backup configuration file,
47 specifically the store section. A few pieces of configuration are taken
48 directly from the user.
49
50 @author: Kenneth J. Pronovici <pronovic@ieee.org>
51 """
52
53
54
55
56
57
58 import sys
59 import os
60 import logging
61 import tempfile
62
63
64 from CedarBackup2.release import AUTHOR, EMAIL, VERSION, DATE, COPYRIGHT
65 from CedarBackup2.util import displayBytes, convertSize, mount, unmount
66 from CedarBackup2.util import UNIT_SECTORS, UNIT_BYTES
67 from CedarBackup2.config import Config
68 from CedarBackup2.filesystem import BackupFileList, compareDigestMaps, normalizeDir
69 from CedarBackup2.cli import Options, setupLogging, setupPathResolver
70 from CedarBackup2.cli import DEFAULT_CONFIG, DEFAULT_LOGFILE, DEFAULT_OWNERSHIP, DEFAULT_MODE
71 from CedarBackup2.actions.constants import STORE_INDICATOR
72 from CedarBackup2.actions.util import createWriter
73 from CedarBackup2.actions.store import writeIndicatorFile
74 from CedarBackup2.actions.util import findDailyDirs
75 from CedarBackup2.util import Diagnostics
76
77
78
79
80
81
82 logger = logging.getLogger("CedarBackup2.log.tools.span")
83
84
85
86
87
88
90
91 """
92 Tool-specific command-line options.
93
94 Most of the cback command-line options are exactly what we need here --
95 logfile path, permissions, verbosity, etc. However, we need to make a few
96 tweaks since we don't accept any actions.
97
98 Also, a few extra command line options that we accept are really ignored
99 underneath. I just don't care about that for a tool like this.
100 """
101
103 """
104 Validates command-line options represented by the object.
105 There are no validations here, because we don't use any actions.
106 @raise ValueError: If one of the validations fails.
107 """
108 pass
109
110
111
112
113
114
115
116
117
118
120 """
121 Implements the command-line interface for the C{cback-span} script.
122
123 Essentially, this is the "main routine" for the cback-span script. It does
124 all of the argument processing for the script, and then also implements the
125 tool functionality.
126
127 This function looks pretty similiar to C{CedarBackup2.cli.cli()}. It's not
128 easy to refactor this code to make it reusable and also readable, so I've
129 decided to just live with the duplication.
130
131 A different error code is returned for each type of failure:
132
133 - C{1}: The Python interpreter version is < 2.7
134 - C{2}: Error processing command-line arguments
135 - C{3}: Error configuring logging
136 - C{4}: Error parsing indicated configuration file
137 - C{5}: Backup was interrupted with a CTRL-C or similar
138 - C{6}: Error executing other parts of the script
139
140 @note: This script uses print rather than logging to the INFO level, because
141 it is interactive. Underlying Cedar Backup functionality uses the logging
142 mechanism exclusively.
143
144 @return: Error code as described above.
145 """
146 try:
147 if map(int, [sys.version_info[0], sys.version_info[1]]) < [2, 7]:
148 sys.stderr.write("Python 2 version 2.7 or greater required.\n")
149 return 1
150 except:
151
152 sys.stderr.write("Python 2 version 2.7 or greater required.\n")
153 return 1
154
155 try:
156 options = SpanOptions(argumentList=sys.argv[1:])
157 except Exception, e:
158 _usage()
159 sys.stderr.write(" *** Error: %s\n" % e)
160 return 2
161
162 if options.help:
163 _usage()
164 return 0
165 if options.version:
166 _version()
167 return 0
168 if options.diagnostics:
169 _diagnostics()
170 return 0
171
172 try:
173 logfile = setupLogging(options)
174 except Exception, e:
175 sys.stderr.write("Error setting up logging: %s\n" % e)
176 return 3
177
178 logger.info("Cedar Backup 'span' utility run started.")
179 logger.info("Options were [%s]", options)
180 logger.info("Logfile is [%s]", logfile)
181
182 if options.config is None:
183 logger.debug("Using default configuration file.")
184 configPath = DEFAULT_CONFIG
185 else:
186 logger.debug("Using user-supplied configuration file.")
187 configPath = options.config
188
189 try:
190 logger.info("Configuration path is [%s]", configPath)
191 config = Config(xmlPath=configPath)
192 setupPathResolver(config)
193 except Exception, e:
194 logger.error("Error reading or handling configuration: %s", e)
195 logger.info("Cedar Backup 'span' utility run completed with status 4.")
196 return 4
197
198 if options.stacktrace:
199 _executeAction(options, config)
200 else:
201 try:
202 _executeAction(options, config)
203 except KeyboardInterrupt:
204 logger.error("Backup interrupted.")
205 logger.info("Cedar Backup 'span' utility run completed with status 5.")
206 return 5
207 except Exception, e:
208 logger.error("Error executing backup: %s", e)
209 logger.info("Cedar Backup 'span' utility run completed with status 6.")
210 return 6
211
212 logger.info("Cedar Backup 'span' utility run completed with status 0.")
213 return 0
214
215
216
217
218
219
220
221
222
223
225 """
226 Prints usage information for the cback script.
227 @param fd: File descriptor used to print information.
228 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
229 """
230 fd.write("\n")
231 fd.write(" Usage: cback-span [switches]\n")
232 fd.write("\n")
233 fd.write(" Cedar Backup 'span' tool.\n")
234 fd.write("\n")
235 fd.write(" This Cedar Backup utility spans staged data between multiple discs.\n")
236 fd.write(" It is a utility, not an extension, and requires user interaction.\n")
237 fd.write("\n")
238 fd.write(" The following switches are accepted, mostly to set up underlying\n")
239 fd.write(" Cedar Backup functionality:\n")
240 fd.write("\n")
241 fd.write(" -h, --help Display this usage/help listing\n")
242 fd.write(" -V, --version Display version information\n")
243 fd.write(" -b, --verbose Print verbose output as well as logging to disk\n")
244 fd.write(" -c, --config Path to config file (default: %s)\n" % DEFAULT_CONFIG)
245 fd.write(" -l, --logfile Path to logfile (default: %s)\n" % DEFAULT_LOGFILE)
246 fd.write(" -o, --owner Logfile ownership, user:group (default: %s:%s)\n" % (DEFAULT_OWNERSHIP[0], DEFAULT_OWNERSHIP[1]))
247 fd.write(" -m, --mode Octal logfile permissions mode (default: %o)\n" % DEFAULT_MODE)
248 fd.write(" -O, --output Record some sub-command (i.e. tar) output to the log\n")
249 fd.write(" -d, --debug Write debugging information to the log (implies --output)\n")
250 fd.write(" -s, --stack Dump a Python stack trace instead of swallowing exceptions\n")
251 fd.write("\n")
252
253
254
255
256
257
259 """
260 Prints version information for the cback script.
261 @param fd: File descriptor used to print information.
262 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
263 """
264 fd.write("\n")
265 fd.write(" Cedar Backup 'span' tool.\n")
266 fd.write(" Included with Cedar Backup version %s, released %s.\n" % (VERSION, DATE))
267 fd.write("\n")
268 fd.write(" Copyright (c) %s %s <%s>.\n" % (COPYRIGHT, AUTHOR, EMAIL))
269 fd.write(" See CREDITS for a list of included code and other contributors.\n")
270 fd.write(" This is free software; there is NO warranty. See the\n")
271 fd.write(" GNU General Public License version 2 for copying conditions.\n")
272 fd.write("\n")
273 fd.write(" Use the --help option for usage information.\n")
274 fd.write("\n")
275
276
277
278
279
280
282 """
283 Prints runtime diagnostics information.
284 @param fd: File descriptor used to print information.
285 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
286 """
287 fd.write("\n")
288 fd.write("Diagnostics:\n")
289 fd.write("\n")
290 Diagnostics().printDiagnostics(fd=fd, prefix=" ")
291 fd.write("\n")
292
293
294
295
296
297
299 """
300 Implements the guts of the cback-span tool.
301
302 @param options: Program command-line options.
303 @type options: SpanOptions object.
304
305 @param config: Program configuration.
306 @type config: Config object.
307
308 @raise Exception: Under many generic error conditions
309 """
310 print ""
311 print "================================================"
312 print " Cedar Backup 'span' tool"
313 print "================================================"
314 print ""
315 print "This the Cedar Backup span tool. It is used to split up staging"
316 print "data when that staging data does not fit onto a single disc."
317 print ""
318 print "This utility operates using Cedar Backup configuration. Configuration"
319 print "specifies which staging directory to look at and which writer device"
320 print "and media type to use."
321 print ""
322 if not _getYesNoAnswer("Continue?", default="Y"):
323 return
324 print "==="
325
326 print ""
327 print "Cedar Backup store configuration looks like this:"
328 print ""
329 print " Source Directory...: %s" % config.store.sourceDir
330 print " Media Type.........: %s" % config.store.mediaType
331 print " Device Type........: %s" % config.store.deviceType
332 print " Device Path........: %s" % config.store.devicePath
333 print " Device SCSI ID.....: %s" % config.store.deviceScsiId
334 print " Drive Speed........: %s" % config.store.driveSpeed
335 print " Check Data Flag....: %s" % config.store.checkData
336 print " No Eject Flag......: %s" % config.store.noEject
337 print ""
338 if not _getYesNoAnswer("Is this OK?", default="Y"):
339 return
340 print "==="
341
342 (writer, mediaCapacity) = _getWriter(config)
343
344 print ""
345 print "Please wait, indexing the source directory (this may take a while)..."
346 (dailyDirs, fileList) = _findDailyDirs(config.store.sourceDir)
347 print "==="
348
349 print ""
350 print "The following daily staging directories have not yet been written to disc:"
351 print ""
352 for dailyDir in dailyDirs:
353 print " %s" % dailyDir
354
355 totalSize = fileList.totalSize()
356 print ""
357 print "The total size of the data in these directories is %s." % displayBytes(totalSize)
358 print ""
359 if not _getYesNoAnswer("Continue?", default="Y"):
360 return
361 print "==="
362
363 print ""
364 print "Based on configuration, the capacity of your media is %s." % displayBytes(mediaCapacity)
365
366 print ""
367 print "Since estimates are not perfect and there is some uncertainly in"
368 print "media capacity calculations, it is good to have a \"cushion\","
369 print "a percentage of capacity to set aside. The cushion reduces the"
370 print "capacity of your media, so a 1.5% cushion leaves 98.5% remaining."
371 print ""
372 cushion = _getFloat("What cushion percentage?", default=4.5)
373 print "==="
374
375 realCapacity = ((100.0 - cushion)/100.0) * mediaCapacity
376 minimumDiscs = (totalSize/realCapacity) + 1
377 print ""
378 print "The real capacity, taking into account the %.2f%% cushion, is %s." % (cushion, displayBytes(realCapacity))
379 print "It will take at least %d disc(s) to store your %s of data." % (minimumDiscs, displayBytes(totalSize))
380 print ""
381 if not _getYesNoAnswer("Continue?", default="Y"):
382 return
383 print "==="
384
385 happy = False
386 while not happy:
387 print ""
388 print "Which algorithm do you want to use to span your data across"
389 print "multiple discs?"
390 print ""
391 print "The following algorithms are available:"
392 print ""
393 print " first....: The \"first-fit\" algorithm"
394 print " best.....: The \"best-fit\" algorithm"
395 print " worst....: The \"worst-fit\" algorithm"
396 print " alternate: The \"alternate-fit\" algorithm"
397 print ""
398 print "If you don't like the results you will have a chance to try a"
399 print "different one later."
400 print ""
401 algorithm = _getChoiceAnswer("Which algorithm?", "worst", [ "first", "best", "worst", "alternate", ])
402 print "==="
403
404 print ""
405 print "Please wait, generating file lists (this may take a while)..."
406 spanSet = fileList.generateSpan(capacity=realCapacity, algorithm="%s_fit" % algorithm)
407 print "==="
408
409 print ""
410 print "Using the \"%s-fit\" algorithm, Cedar Backup can split your data" % algorithm
411 print "into %d discs." % len(spanSet)
412 print ""
413 counter = 0
414 for item in spanSet:
415 counter += 1
416 print "Disc %d: %d files, %s, %.2f%% utilization" % (counter, len(item.fileList),
417 displayBytes(item.size), item.utilization)
418 print ""
419 if _getYesNoAnswer("Accept this solution?", default="Y"):
420 happy = True
421 print "==="
422
423 counter = 0
424 for spanItem in spanSet:
425 counter += 1
426 if counter == 1:
427 print ""
428 _getReturn("Please place the first disc in your backup device.\nPress return when ready.")
429 print "==="
430 else:
431 print ""
432 _getReturn("Please replace the disc in your backup device.\nPress return when ready.")
433 print "==="
434 _writeDisc(config, writer, spanItem)
435
436 _writeStoreIndicator(config, dailyDirs)
437
438 print ""
439 print "Completed writing all discs."
440
441
442
443
444
445
447 """
448 Returns a list of all daily staging directories that have not yet been
449 stored.
450
451 The store indicator file C{cback.store} will be written to a daily staging
452 directory once that directory is written to disc. So, this function looks
453 at each daily staging directory within the configured staging directory, and
454 returns a list of those which do not contain the indicator file.
455
456 Returned is a tuple containing two items: a list of daily staging
457 directories, and a BackupFileList containing all files among those staging
458 directories.
459
460 @param stagingDir: Configured staging directory
461
462 @return: Tuple (staging dirs, backup file list)
463 """
464 results = findDailyDirs(stagingDir, STORE_INDICATOR)
465 fileList = BackupFileList()
466 for item in results:
467 fileList.addDirContents(item)
468 return (results, fileList)
469
470
471
472
473
474
486
487
488
489
490
491
502
503
504
505
506
507
521
523 """
524 Initialize an ISO image for a span item.
525 @param config: Cedar Backup configuration
526 @param writer: Writer to use
527 @param spanItem: Span item to write
528 """
529 complete = False
530 while not complete:
531 try:
532 print "Initializing image..."
533 writer.initializeImage(newDisc=True, tmpdir=config.options.workingDir)
534 for path in spanItem.fileList:
535 graftPoint = os.path.dirname(path.replace(config.store.sourceDir, "", 1))
536 writer.addImageEntry(path, graftPoint)
537 complete = True
538 except KeyboardInterrupt, e:
539 raise e
540 except Exception, e:
541 logger.error("Failed to initialize image: %s", e)
542 if not _getYesNoAnswer("Retry initialization step?", default="Y"):
543 raise e
544 print "Ok, attempting retry."
545 print "==="
546 print "Completed initializing image."
547
549 """
550 Writes a ISO image for a span item.
551 @param config: Cedar Backup configuration
552 @param writer: Writer to use
553 """
554 complete = False
555 while not complete:
556 try:
557 print "Writing image to disc..."
558 writer.writeImage()
559 complete = True
560 except KeyboardInterrupt, e:
561 raise e
562 except Exception, e:
563 logger.error("Failed to write image: %s", e)
564 if not _getYesNoAnswer("Retry this step?", default="Y"):
565 raise e
566 print "Ok, attempting retry."
567 _getReturn("Please replace media if needed.\nPress return when ready.")
568 print "==="
569 print "Completed writing image."
570
572 """
573 Run a consistency check on an ISO image for a span item.
574 @param config: Cedar Backup configuration
575 @param writer: Writer to use
576 @param spanItem: Span item to write
577 """
578 if config.store.checkData:
579 complete = False
580 while not complete:
581 try:
582 print "Running consistency check..."
583 _consistencyCheck(config, spanItem.fileList)
584 complete = True
585 except KeyboardInterrupt, e:
586 raise e
587 except Exception, e:
588 logger.error("Consistency check failed: %s", e)
589 if not _getYesNoAnswer("Retry the consistency check?", default="Y"):
590 raise e
591 if _getYesNoAnswer("Rewrite the disc first?", default="N"):
592 print "Ok, attempting retry."
593 _getReturn("Please replace the disc in your backup device.\nPress return when ready.")
594 print "==="
595 _discWriteImage(config, writer)
596 else:
597 print "Ok, attempting retry."
598 print "==="
599 print "Completed consistency check."
600
601
602
603
604
605
607 """
608 Runs a consistency check against media in the backup device.
609
610 The function mounts the device at a temporary mount point in the working
611 directory, and then compares the passed-in file list's digest map with the
612 one generated from the disc. The two lists should be identical.
613
614 If no exceptions are thrown, there were no problems with the consistency
615 check.
616
617 @warning: The implementation of this function is very UNIX-specific.
618
619 @param config: Config object.
620 @param fileList: BackupFileList whose contents to check against
621
622 @raise ValueError: If the check fails
623 @raise IOError: If there is a problem working with the media.
624 """
625 logger.debug("Running consistency check.")
626 mountPoint = tempfile.mkdtemp(dir=config.options.workingDir)
627 try:
628 mount(config.store.devicePath, mountPoint, "iso9660")
629 discList = BackupFileList()
630 discList.addDirContents(mountPoint)
631 sourceList = BackupFileList()
632 sourceList.extend(fileList)
633 discListDigest = discList.generateDigestMap(stripPrefix=normalizeDir(mountPoint))
634 sourceListDigest = sourceList.generateDigestMap(stripPrefix=normalizeDir(config.store.sourceDir))
635 compareDigestMaps(sourceListDigest, discListDigest, verbose=True)
636 logger.info("Consistency check completed. No problems found.")
637 finally:
638 unmount(mountPoint, True, 5, 1)
639
640
641
642
643
644
646 """
647 Get a yes/no answer from the user.
648 The default will be placed at the end of the prompt.
649 A "Y" or "y" is considered yes, anything else no.
650 A blank (empty) response results in the default.
651 @param prompt: Prompt to show.
652 @param default: Default to set if the result is blank
653 @return: Boolean true/false corresponding to Y/N
654 """
655 if default == "Y":
656 prompt = "%s [Y/n]: " % prompt
657 else:
658 prompt = "%s [y/N]: " % prompt
659 answer = raw_input(prompt)
660 if answer in [ None, "", ]:
661 answer = default
662 if answer[0] in [ "Y", "y", ]:
663 return True
664 else:
665 return False
666
668 """
669 Get a particular choice from the user.
670 The default will be placed at the end of the prompt.
671 The function loops until getting a valid choice.
672 A blank (empty) response results in the default.
673 @param prompt: Prompt to show.
674 @param default: Default to set if the result is None or blank.
675 @param validChoices: List of valid choices (strings)
676 @return: Valid choice from user.
677 """
678 prompt = "%s [%s]: " % (prompt, default)
679 answer = raw_input(prompt)
680 if answer in [ None, "", ]:
681 answer = default
682 while answer not in validChoices:
683 print "Choice must be one of %s" % validChoices
684 answer = raw_input(prompt)
685 return answer
686
688 """
689 Get a floating point number from the user.
690 The default will be placed at the end of the prompt.
691 The function loops until getting a valid floating point number.
692 A blank (empty) response results in the default.
693 @param prompt: Prompt to show.
694 @param default: Default to set if the result is None or blank.
695 @return: Floating point number from user
696 """
697 prompt = "%s [%.2f]: " % (prompt, default)
698 while True:
699 answer = raw_input(prompt)
700 if answer in [ None, "" ]:
701 return default
702 else:
703 try:
704 return float(answer)
705 except ValueError:
706 print "Enter a floating point number."
707
709 """
710 Get a return key from the user.
711 @param prompt: Prompt to show.
712 """
713 raw_input(prompt)
714
715
716
717
718
719
720 if __name__ == "__main__":
721 sys.exit(cli())
722