1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 """
39 Provides an extension to back up mbox email files.
40
41 Backing up email
42 ================
43
44 Email folders (often stored as mbox flatfiles) are not well-suited being backed
45 up with an incremental backup like the one offered by Cedar Backup. This is
46 because mbox files often change on a daily basis, forcing the incremental
47 backup process to back them up every day in order to avoid losing data. This
48 can result in quite a bit of wasted space when backing up large folders. (Note
49 that the alternative maildir format does not share this problem, since it
50 typically uses one file per message.)
51
52 One solution to this problem is to design a smarter incremental backup process,
53 which backs up baseline content on the first day of the week, and then backs up
54 only new messages added to that folder on every other day of the week. This way,
55 the backup for any single day is only as large as the messages placed into the
56 folder on that day. The backup isn't as "perfect" as the incremental backup
57 process, because it doesn't preserve information about messages deleted from
58 the backed-up folder. However, it should be much more space-efficient, and
59 in a recovery situation, it seems better to restore too much data rather
60 than too little.
61
62 What is this extension?
63 =======================
64
65 This is a Cedar Backup extension used to back up mbox email files via the Cedar
66 Backup command line. Individual mbox files or directories containing mbox
67 files can be backed up using the same collect modes allowed for filesystems in
68 the standard Cedar Backup collect action: weekly, daily, incremental. It
69 implements the "smart" incremental backup process discussed above, using
70 functionality provided by the C{grepmail} utility.
71
72 This extension requires a new configuration section <mbox> and is intended to
73 be run either immediately before or immediately after the standard collect
74 action. Aside from its own configuration, it requires the options and collect
75 configuration sections in the standard Cedar Backup configuration file.
76
77 The mbox action is conceptually similar to the standard collect action,
78 except that mbox directories are not collected recursively. This implies
79 some configuration changes (i.e. there's no need for global exclusions or an
80 ignore file). If you back up a directory, all of the mbox files in that
81 directory are backed up into a single tar file using the indicated
82 compression method.
83
84 @author: Kenneth J. Pronovici <pronovic@ieee.org>
85 """
86
87
88
89
90
91
92 import os
93 import logging
94 import datetime
95 import pickle
96 import tempfile
97 from bz2 import BZ2File
98 from gzip import GzipFile
99
100
101 from CedarBackup2.filesystem import FilesystemList, BackupFileList
102 from CedarBackup2.xmlutil import createInputDom, addContainerNode, addStringNode
103 from CedarBackup2.xmlutil import isElement, readChildren, readFirstChild, readString, readStringList
104 from CedarBackup2.config import VALID_COLLECT_MODES, VALID_COMPRESS_MODES
105 from CedarBackup2.util import isStartOfWeek, buildNormalizedPath
106 from CedarBackup2.util import resolveCommand, executeCommand
107 from CedarBackup2.util import ObjectTypeList, UnorderedList, RegexList, encodePath, changeOwnership
108
109
110
111
112
113
114 logger = logging.getLogger("CedarBackup2.log.extend.mbox")
115
116 GREPMAIL_COMMAND = [ "grepmail", ]
117 REVISION_PATH_EXTENSION = "mboxlast"
118
119
120
121
122
123
124 -class MboxFile(object):
125
126 """
127 Class representing mbox file configuration..
128
129 The following restrictions exist on data in this class:
130
131 - The absolute path must be absolute.
132 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
133 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
134
135 @sort: __init__, __repr__, __str__, __cmp__, absolutePath, collectMode, compressMode
136 """
137
138 - def __init__(self, absolutePath=None, collectMode=None, compressMode=None):
139 """
140 Constructor for the C{MboxFile} class.
141
142 You should never directly instantiate this class.
143
144 @param absolutePath: Absolute path to an mbox file on disk.
145 @param collectMode: Overridden collect mode for this directory.
146 @param compressMode: Overridden compression mode for this directory.
147 """
148 self._absolutePath = None
149 self._collectMode = None
150 self._compressMode = None
151 self.absolutePath = absolutePath
152 self.collectMode = collectMode
153 self.compressMode = compressMode
154
160
162 """
163 Informal string representation for class instance.
164 """
165 return self.__repr__()
166
191
193 """
194 Property target used to set the absolute path.
195 The value must be an absolute path if it is not C{None}.
196 It does not have to exist on disk at the time of assignment.
197 @raise ValueError: If the value is not an absolute path.
198 @raise ValueError: If the value cannot be encoded properly.
199 """
200 if value is not None:
201 if not os.path.isabs(value):
202 raise ValueError("Absolute path must be, er, an absolute path.")
203 self._absolutePath = encodePath(value)
204
206 """
207 Property target used to get the absolute path.
208 """
209 return self._absolutePath
210
212 """
213 Property target used to set the collect mode.
214 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
215 @raise ValueError: If the value is not valid.
216 """
217 if value is not None:
218 if value not in VALID_COLLECT_MODES:
219 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
220 self._collectMode = value
221
223 """
224 Property target used to get the collect mode.
225 """
226 return self._collectMode
227
229 """
230 Property target used to set the compress mode.
231 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
232 @raise ValueError: If the value is not valid.
233 """
234 if value is not None:
235 if value not in VALID_COMPRESS_MODES:
236 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
237 self._compressMode = value
238
240 """
241 Property target used to get the compress mode.
242 """
243 return self._compressMode
244
245 absolutePath = property(_getAbsolutePath, _setAbsolutePath, None, doc="Absolute path to the mbox file.")
246 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this mbox file.")
247 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this mbox file.")
248
249
250
251
252
253
254 -class MboxDir(object):
255
256 """
257 Class representing mbox directory configuration..
258
259 The following restrictions exist on data in this class:
260
261 - The absolute path must be absolute.
262 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
263 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
264
265 Unlike collect directory configuration, this is the only place exclusions
266 are allowed (no global exclusions at the <mbox> configuration level). Also,
267 we only allow relative exclusions and there is no configured ignore file.
268 This is because mbox directory backups are not recursive.
269
270 @sort: __init__, __repr__, __str__, __cmp__, absolutePath, collectMode,
271 compressMode, relativeExcludePaths, excludePatterns
272 """
273
274 - def __init__(self, absolutePath=None, collectMode=None, compressMode=None,
275 relativeExcludePaths=None, excludePatterns=None):
276 """
277 Constructor for the C{MboxDir} class.
278
279 You should never directly instantiate this class.
280
281 @param absolutePath: Absolute path to a mbox file on disk.
282 @param collectMode: Overridden collect mode for this directory.
283 @param compressMode: Overridden compression mode for this directory.
284 @param relativeExcludePaths: List of relative paths to exclude.
285 @param excludePatterns: List of regular expression patterns to exclude
286 """
287 self._absolutePath = None
288 self._collectMode = None
289 self._compressMode = None
290 self._relativeExcludePaths = None
291 self._excludePatterns = None
292 self.absolutePath = absolutePath
293 self.collectMode = collectMode
294 self.compressMode = compressMode
295 self.relativeExcludePaths = relativeExcludePaths
296 self.excludePatterns = excludePatterns
297
304
306 """
307 Informal string representation for class instance.
308 """
309 return self.__repr__()
310
345
347 """
348 Property target used to set the absolute path.
349 The value must be an absolute path if it is not C{None}.
350 It does not have to exist on disk at the time of assignment.
351 @raise ValueError: If the value is not an absolute path.
352 @raise ValueError: If the value cannot be encoded properly.
353 """
354 if value is not None:
355 if not os.path.isabs(value):
356 raise ValueError("Absolute path must be, er, an absolute path.")
357 self._absolutePath = encodePath(value)
358
360 """
361 Property target used to get the absolute path.
362 """
363 return self._absolutePath
364
366 """
367 Property target used to set the collect mode.
368 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
369 @raise ValueError: If the value is not valid.
370 """
371 if value is not None:
372 if value not in VALID_COLLECT_MODES:
373 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
374 self._collectMode = value
375
377 """
378 Property target used to get the collect mode.
379 """
380 return self._collectMode
381
383 """
384 Property target used to set the compress mode.
385 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
386 @raise ValueError: If the value is not valid.
387 """
388 if value is not None:
389 if value not in VALID_COMPRESS_MODES:
390 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
391 self._compressMode = value
392
394 """
395 Property target used to get the compress mode.
396 """
397 return self._compressMode
398
400 """
401 Property target used to set the relative exclude paths list.
402 Elements do not have to exist on disk at the time of assignment.
403 """
404 if value is None:
405 self._relativeExcludePaths = None
406 else:
407 try:
408 saved = self._relativeExcludePaths
409 self._relativeExcludePaths = UnorderedList()
410 self._relativeExcludePaths.extend(value)
411 except Exception, e:
412 self._relativeExcludePaths = saved
413 raise e
414
416 """
417 Property target used to get the relative exclude paths list.
418 """
419 return self._relativeExcludePaths
420
422 """
423 Property target used to set the exclude patterns list.
424 """
425 if value is None:
426 self._excludePatterns = None
427 else:
428 try:
429 saved = self._excludePatterns
430 self._excludePatterns = RegexList()
431 self._excludePatterns.extend(value)
432 except Exception, e:
433 self._excludePatterns = saved
434 raise e
435
437 """
438 Property target used to get the exclude patterns list.
439 """
440 return self._excludePatterns
441
442 absolutePath = property(_getAbsolutePath, _setAbsolutePath, None, doc="Absolute path to the mbox directory.")
443 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this mbox directory.")
444 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this mbox directory.")
445 relativeExcludePaths = property(_getRelativeExcludePaths, _setRelativeExcludePaths, None, "List of relative paths to exclude.")
446 excludePatterns = property(_getExcludePatterns, _setExcludePatterns, None, "List of regular expression patterns to exclude.")
447
454
455 """
456 Class representing mbox configuration.
457
458 Mbox configuration is used for backing up mbox email files.
459
460 The following restrictions exist on data in this class:
461
462 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
463 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
464 - The C{mboxFiles} list must be a list of C{MboxFile} objects
465 - The C{mboxDirs} list must be a list of C{MboxDir} objects
466
467 For the C{mboxFiles} and C{mboxDirs} lists, validation is accomplished
468 through the L{util.ObjectTypeList} list implementation that overrides common
469 list methods and transparently ensures that each element is of the proper
470 type.
471
472 Unlike collect configuration, no global exclusions are allowed on this
473 level. We only allow relative exclusions at the mbox directory level.
474 Also, there is no configured ignore file. This is because mbox directory
475 backups are not recursive.
476
477 @note: Lists within this class are "unordered" for equality comparisons.
478
479 @sort: __init__, __repr__, __str__, __cmp__, collectMode, compressMode, mboxFiles, mboxDirs
480 """
481
482 - def __init__(self, collectMode=None, compressMode=None, mboxFiles=None, mboxDirs=None):
483 """
484 Constructor for the C{MboxConfig} class.
485
486 @param collectMode: Default collect mode.
487 @param compressMode: Default compress mode.
488 @param mboxFiles: List of mbox files to back up
489 @param mboxDirs: List of mbox directories to back up
490
491 @raise ValueError: If one of the values is invalid.
492 """
493 self._collectMode = None
494 self._compressMode = None
495 self._mboxFiles = None
496 self._mboxDirs = None
497 self.collectMode = collectMode
498 self.compressMode = compressMode
499 self.mboxFiles = mboxFiles
500 self.mboxDirs = mboxDirs
501
507
509 """
510 Informal string representation for class instance.
511 """
512 return self.__repr__()
513
515 """
516 Definition of equals operator for this class.
517 Lists within this class are "unordered" for equality comparisons.
518 @param other: Other object to compare to.
519 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
520 """
521 if other is None:
522 return 1
523 if self.collectMode != other.collectMode:
524 if self.collectMode < other.collectMode:
525 return -1
526 else:
527 return 1
528 if self.compressMode != other.compressMode:
529 if self.compressMode < other.compressMode:
530 return -1
531 else:
532 return 1
533 if self.mboxFiles != other.mboxFiles:
534 if self.mboxFiles < other.mboxFiles:
535 return -1
536 else:
537 return 1
538 if self.mboxDirs != other.mboxDirs:
539 if self.mboxDirs < other.mboxDirs:
540 return -1
541 else:
542 return 1
543 return 0
544
546 """
547 Property target used to set the collect mode.
548 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
549 @raise ValueError: If the value is not valid.
550 """
551 if value is not None:
552 if value not in VALID_COLLECT_MODES:
553 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
554 self._collectMode = value
555
557 """
558 Property target used to get the collect mode.
559 """
560 return self._collectMode
561
563 """
564 Property target used to set the compress mode.
565 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
566 @raise ValueError: If the value is not valid.
567 """
568 if value is not None:
569 if value not in VALID_COMPRESS_MODES:
570 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
571 self._compressMode = value
572
574 """
575 Property target used to get the compress mode.
576 """
577 return self._compressMode
578
580 """
581 Property target used to set the mboxFiles list.
582 Either the value must be C{None} or each element must be an C{MboxFile}.
583 @raise ValueError: If the value is not an C{MboxFile}
584 """
585 if value is None:
586 self._mboxFiles = None
587 else:
588 try:
589 saved = self._mboxFiles
590 self._mboxFiles = ObjectTypeList(MboxFile, "MboxFile")
591 self._mboxFiles.extend(value)
592 except Exception, e:
593 self._mboxFiles = saved
594 raise e
595
597 """
598 Property target used to get the mboxFiles list.
599 """
600 return self._mboxFiles
601
603 """
604 Property target used to set the mboxDirs list.
605 Either the value must be C{None} or each element must be an C{MboxDir}.
606 @raise ValueError: If the value is not an C{MboxDir}
607 """
608 if value is None:
609 self._mboxDirs = None
610 else:
611 try:
612 saved = self._mboxDirs
613 self._mboxDirs = ObjectTypeList(MboxDir, "MboxDir")
614 self._mboxDirs.extend(value)
615 except Exception, e:
616 self._mboxDirs = saved
617 raise e
618
620 """
621 Property target used to get the mboxDirs list.
622 """
623 return self._mboxDirs
624
625 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Default collect mode.")
626 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Default compress mode.")
627 mboxFiles = property(_getMboxFiles, _setMboxFiles, None, doc="List of mbox files to back up.")
628 mboxDirs = property(_getMboxDirs, _setMboxDirs, None, doc="List of mbox directories to back up.")
629
636
637 """
638 Class representing this extension's configuration document.
639
640 This is not a general-purpose configuration object like the main Cedar
641 Backup configuration object. Instead, it just knows how to parse and emit
642 Mbox-specific configuration values. Third parties who need to read and
643 write configuration related to this extension should access it through the
644 constructor, C{validate} and C{addConfig} methods.
645
646 @note: Lists within this class are "unordered" for equality comparisons.
647
648 @sort: __init__, __repr__, __str__, __cmp__, mbox, validate, addConfig
649 """
650
651 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
652 """
653 Initializes a configuration object.
654
655 If you initialize the object without passing either C{xmlData} or
656 C{xmlPath} then configuration will be empty and will be invalid until it
657 is filled in properly.
658
659 No reference to the original XML data or original path is saved off by
660 this class. Once the data has been parsed (successfully or not) this
661 original information is discarded.
662
663 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
664 method will be called (with its default arguments) against configuration
665 after successfully parsing any passed-in XML. Keep in mind that even if
666 C{validate} is C{False}, it might not be possible to parse the passed-in
667 XML document if lower-level validations fail.
668
669 @note: It is strongly suggested that the C{validate} option always be set
670 to C{True} (the default) unless there is a specific need to read in
671 invalid configuration from disk.
672
673 @param xmlData: XML data representing configuration.
674 @type xmlData: String data.
675
676 @param xmlPath: Path to an XML file on disk.
677 @type xmlPath: Absolute path to a file on disk.
678
679 @param validate: Validate the document after parsing it.
680 @type validate: Boolean true/false.
681
682 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
683 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
684 @raise ValueError: If the parsed configuration document is not valid.
685 """
686 self._mbox = None
687 self.mbox = None
688 if xmlData is not None and xmlPath is not None:
689 raise ValueError("Use either xmlData or xmlPath, but not both.")
690 if xmlData is not None:
691 self._parseXmlData(xmlData)
692 if validate:
693 self.validate()
694 elif xmlPath is not None:
695 xmlData = open(xmlPath).read()
696 self._parseXmlData(xmlData)
697 if validate:
698 self.validate()
699
701 """
702 Official string representation for class instance.
703 """
704 return "LocalConfig(%s)" % (self.mbox)
705
707 """
708 Informal string representation for class instance.
709 """
710 return self.__repr__()
711
713 """
714 Definition of equals operator for this class.
715 Lists within this class are "unordered" for equality comparisons.
716 @param other: Other object to compare to.
717 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
718 """
719 if other is None:
720 return 1
721 if self.mbox != other.mbox:
722 if self.mbox < other.mbox:
723 return -1
724 else:
725 return 1
726 return 0
727
729 """
730 Property target used to set the mbox configuration value.
731 If not C{None}, the value must be a C{MboxConfig} object.
732 @raise ValueError: If the value is not a C{MboxConfig}
733 """
734 if value is None:
735 self._mbox = None
736 else:
737 if not isinstance(value, MboxConfig):
738 raise ValueError("Value must be a C{MboxConfig} object.")
739 self._mbox = value
740
742 """
743 Property target used to get the mbox configuration value.
744 """
745 return self._mbox
746
747 mbox = property(_getMbox, _setMbox, None, "Mbox configuration in terms of a C{MboxConfig} object.")
748
750 """
751 Validates configuration represented by the object.
752
753 Mbox configuration must be filled in. Within that, the collect mode and
754 compress mode are both optional, but the list of repositories must
755 contain at least one entry.
756
757 Each configured file or directory must contain an absolute path, and then
758 must be either able to take collect mode and compress mode configuration
759 from the parent C{MboxConfig} object, or must set each value on its own.
760
761 @raise ValueError: If one of the validations fails.
762 """
763 if self.mbox is None:
764 raise ValueError("Mbox section is required.")
765 if (self.mbox.mboxFiles is None or len(self.mbox.mboxFiles) < 1) and \
766 (self.mbox.mboxDirs is None or len(self.mbox.mboxDirs) < 1):
767 raise ValueError("At least one mbox file or directory must be configured.")
768 if self.mbox.mboxFiles is not None:
769 for mboxFile in self.mbox.mboxFiles:
770 if mboxFile.absolutePath is None:
771 raise ValueError("Each mbox file must set an absolute path.")
772 if self.mbox.collectMode is None and mboxFile.collectMode is None:
773 raise ValueError("Collect mode must either be set in parent mbox section or individual mbox file.")
774 if self.mbox.compressMode is None and mboxFile.compressMode is None:
775 raise ValueError("Compress mode must either be set in parent mbox section or individual mbox file.")
776 if self.mbox.mboxDirs is not None:
777 for mboxDir in self.mbox.mboxDirs:
778 if mboxDir.absolutePath is None:
779 raise ValueError("Each mbox directory must set an absolute path.")
780 if self.mbox.collectMode is None and mboxDir.collectMode is None:
781 raise ValueError("Collect mode must either be set in parent mbox section or individual mbox directory.")
782 if self.mbox.compressMode is None and mboxDir.compressMode is None:
783 raise ValueError("Compress mode must either be set in parent mbox section or individual mbox directory.")
784
786 """
787 Adds an <mbox> configuration section as the next child of a parent.
788
789 Third parties should use this function to write configuration related to
790 this extension.
791
792 We add the following fields to the document::
793
794 collectMode //cb_config/mbox/collectMode
795 compressMode //cb_config/mbox/compressMode
796
797 We also add groups of the following items, one list element per
798 item::
799
800 mboxFiles //cb_config/mbox/file
801 mboxDirs //cb_config/mbox/dir
802
803 The mbox files and mbox directories are added by L{_addMboxFile} and
804 L{_addMboxDir}.
805
806 @param xmlDom: DOM tree as from C{impl.createDocument()}.
807 @param parentNode: Parent that the section should be appended to.
808 """
809 if self.mbox is not None:
810 sectionNode = addContainerNode(xmlDom, parentNode, "mbox")
811 addStringNode(xmlDom, sectionNode, "collect_mode", self.mbox.collectMode)
812 addStringNode(xmlDom, sectionNode, "compress_mode", self.mbox.compressMode)
813 if self.mbox.mboxFiles is not None:
814 for mboxFile in self.mbox.mboxFiles:
815 LocalConfig._addMboxFile(xmlDom, sectionNode, mboxFile)
816 if self.mbox.mboxDirs is not None:
817 for mboxDir in self.mbox.mboxDirs:
818 LocalConfig._addMboxDir(xmlDom, sectionNode, mboxDir)
819
821 """
822 Internal method to parse an XML string into the object.
823
824 This method parses the XML document into a DOM tree (C{xmlDom}) and then
825 calls a static method to parse the mbox configuration section.
826
827 @param xmlData: XML data to be parsed
828 @type xmlData: String data
829
830 @raise ValueError: If the XML cannot be successfully parsed.
831 """
832 (xmlDom, parentNode) = createInputDom(xmlData)
833 self._mbox = LocalConfig._parseMbox(parentNode)
834
835 @staticmethod
837 """
838 Parses an mbox configuration section.
839
840 We read the following individual fields::
841
842 collectMode //cb_config/mbox/collect_mode
843 compressMode //cb_config/mbox/compress_mode
844
845 We also read groups of the following item, one list element per
846 item::
847
848 mboxFiles //cb_config/mbox/file
849 mboxDirs //cb_config/mbox/dir
850
851 The mbox files are parsed by L{_parseMboxFiles} and the mbox
852 directories are parsed by L{_parseMboxDirs}.
853
854 @param parent: Parent node to search beneath.
855
856 @return: C{MboxConfig} object or C{None} if the section does not exist.
857 @raise ValueError: If some filled-in value is invalid.
858 """
859 mbox = None
860 section = readFirstChild(parent, "mbox")
861 if section is not None:
862 mbox = MboxConfig()
863 mbox.collectMode = readString(section, "collect_mode")
864 mbox.compressMode = readString(section, "compress_mode")
865 mbox.mboxFiles = LocalConfig._parseMboxFiles(section)
866 mbox.mboxDirs = LocalConfig._parseMboxDirs(section)
867 return mbox
868
869 @staticmethod
871 """
872 Reads a list of C{MboxFile} objects from immediately beneath the parent.
873
874 We read the following individual fields::
875
876 absolutePath abs_path
877 collectMode collect_mode
878 compressMode compess_mode
879
880 @param parent: Parent node to search beneath.
881
882 @return: List of C{MboxFile} objects or C{None} if none are found.
883 @raise ValueError: If some filled-in value is invalid.
884 """
885 lst = []
886 for entry in readChildren(parent, "file"):
887 if isElement(entry):
888 mboxFile = MboxFile()
889 mboxFile.absolutePath = readString(entry, "abs_path")
890 mboxFile.collectMode = readString(entry, "collect_mode")
891 mboxFile.compressMode = readString(entry, "compress_mode")
892 lst.append(mboxFile)
893 if lst == []:
894 lst = None
895 return lst
896
897 @staticmethod
899 """
900 Reads a list of C{MboxDir} objects from immediately beneath the parent.
901
902 We read the following individual fields::
903
904 absolutePath abs_path
905 collectMode collect_mode
906 compressMode compess_mode
907
908 We also read groups of the following items, one list element per
909 item::
910
911 relativeExcludePaths exclude/rel_path
912 excludePatterns exclude/pattern
913
914 The exclusions are parsed by L{_parseExclusions}.
915
916 @param parent: Parent node to search beneath.
917
918 @return: List of C{MboxDir} objects or C{None} if none are found.
919 @raise ValueError: If some filled-in value is invalid.
920 """
921 lst = []
922 for entry in readChildren(parent, "dir"):
923 if isElement(entry):
924 mboxDir = MboxDir()
925 mboxDir.absolutePath = readString(entry, "abs_path")
926 mboxDir.collectMode = readString(entry, "collect_mode")
927 mboxDir.compressMode = readString(entry, "compress_mode")
928 (mboxDir.relativeExcludePaths, mboxDir.excludePatterns) = LocalConfig._parseExclusions(entry)
929 lst.append(mboxDir)
930 if lst == []:
931 lst = None
932 return lst
933
934 @staticmethod
936 """
937 Reads exclusions data from immediately beneath the parent.
938
939 We read groups of the following items, one list element per item::
940
941 relative exclude/rel_path
942 patterns exclude/pattern
943
944 If there are none of some pattern (i.e. no relative path items) then
945 C{None} will be returned for that item in the tuple.
946
947 @param parentNode: Parent node to search beneath.
948
949 @return: Tuple of (relative, patterns) exclusions.
950 """
951 section = readFirstChild(parentNode, "exclude")
952 if section is None:
953 return (None, None)
954 else:
955 relative = readStringList(section, "rel_path")
956 patterns = readStringList(section, "pattern")
957 return (relative, patterns)
958
959 @staticmethod
961 """
962 Adds an mbox file container as the next child of a parent.
963
964 We add the following fields to the document::
965
966 absolutePath file/abs_path
967 collectMode file/collect_mode
968 compressMode file/compress_mode
969
970 The <file> node itself is created as the next child of the parent node.
971 This method only adds one mbox file node. The parent must loop for each
972 mbox file in the C{MboxConfig} object.
973
974 If C{mboxFile} is C{None}, this method call will be a no-op.
975
976 @param xmlDom: DOM tree as from C{impl.createDocument()}.
977 @param parentNode: Parent that the section should be appended to.
978 @param mboxFile: MboxFile to be added to the document.
979 """
980 if mboxFile is not None:
981 sectionNode = addContainerNode(xmlDom, parentNode, "file")
982 addStringNode(xmlDom, sectionNode, "abs_path", mboxFile.absolutePath)
983 addStringNode(xmlDom, sectionNode, "collect_mode", mboxFile.collectMode)
984 addStringNode(xmlDom, sectionNode, "compress_mode", mboxFile.compressMode)
985
986 @staticmethod
988 """
989 Adds an mbox directory container as the next child of a parent.
990
991 We add the following fields to the document::
992
993 absolutePath dir/abs_path
994 collectMode dir/collect_mode
995 compressMode dir/compress_mode
996
997 We also add groups of the following items, one list element per item::
998
999 relativeExcludePaths dir/exclude/rel_path
1000 excludePatterns dir/exclude/pattern
1001
1002 The <dir> node itself is created as the next child of the parent node.
1003 This method only adds one mbox directory node. The parent must loop for
1004 each mbox directory in the C{MboxConfig} object.
1005
1006 If C{mboxDir} is C{None}, this method call will be a no-op.
1007
1008 @param xmlDom: DOM tree as from C{impl.createDocument()}.
1009 @param parentNode: Parent that the section should be appended to.
1010 @param mboxDir: MboxDir to be added to the document.
1011 """
1012 if mboxDir is not None:
1013 sectionNode = addContainerNode(xmlDom, parentNode, "dir")
1014 addStringNode(xmlDom, sectionNode, "abs_path", mboxDir.absolutePath)
1015 addStringNode(xmlDom, sectionNode, "collect_mode", mboxDir.collectMode)
1016 addStringNode(xmlDom, sectionNode, "compress_mode", mboxDir.compressMode)
1017 if ((mboxDir.relativeExcludePaths is not None and mboxDir.relativeExcludePaths != []) or
1018 (mboxDir.excludePatterns is not None and mboxDir.excludePatterns != [])):
1019 excludeNode = addContainerNode(xmlDom, sectionNode, "exclude")
1020 if mboxDir.relativeExcludePaths is not None:
1021 for relativePath in mboxDir.relativeExcludePaths:
1022 addStringNode(xmlDom, excludeNode, "rel_path", relativePath)
1023 if mboxDir.excludePatterns is not None:
1024 for pattern in mboxDir.excludePatterns:
1025 addStringNode(xmlDom, excludeNode, "pattern", pattern)
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036 -def executeAction(configPath, options, config):
1037 """
1038 Executes the mbox backup action.
1039
1040 @param configPath: Path to configuration file on disk.
1041 @type configPath: String representing a path on disk.
1042
1043 @param options: Program command-line options.
1044 @type options: Options object.
1045
1046 @param config: Program configuration.
1047 @type config: Config object.
1048
1049 @raise ValueError: Under many generic error conditions
1050 @raise IOError: If a backup could not be written for some reason.
1051 """
1052 logger.debug("Executing mbox extended action.")
1053 newRevision = datetime.datetime.today()
1054 if config.options is None or config.collect is None:
1055 raise ValueError("Cedar Backup configuration is not properly filled in.")
1056 local = LocalConfig(xmlPath=configPath)
1057 todayIsStart = isStartOfWeek(config.options.startingDay)
1058 fullBackup = options.full or todayIsStart
1059 logger.debug("Full backup flag is [%s]", fullBackup)
1060 if local.mbox.mboxFiles is not None:
1061 for mboxFile in local.mbox.mboxFiles:
1062 logger.debug("Working with mbox file [%s]", mboxFile.absolutePath)
1063 collectMode = _getCollectMode(local, mboxFile)
1064 compressMode = _getCompressMode(local, mboxFile)
1065 lastRevision = _loadLastRevision(config, mboxFile, fullBackup, collectMode)
1066 if fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart):
1067 logger.debug("Mbox file meets criteria to be backed up today.")
1068 _backupMboxFile(config, mboxFile.absolutePath, fullBackup,
1069 collectMode, compressMode, lastRevision, newRevision)
1070 else:
1071 logger.debug("Mbox file will not be backed up, per collect mode.")
1072 if collectMode == 'incr':
1073 _writeNewRevision(config, mboxFile, newRevision)
1074 if local.mbox.mboxDirs is not None:
1075 for mboxDir in local.mbox.mboxDirs:
1076 logger.debug("Working with mbox directory [%s]", mboxDir.absolutePath)
1077 collectMode = _getCollectMode(local, mboxDir)
1078 compressMode = _getCompressMode(local, mboxDir)
1079 lastRevision = _loadLastRevision(config, mboxDir, fullBackup, collectMode)
1080 (excludePaths, excludePatterns) = _getExclusions(mboxDir)
1081 if fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart):
1082 logger.debug("Mbox directory meets criteria to be backed up today.")
1083 _backupMboxDir(config, mboxDir.absolutePath,
1084 fullBackup, collectMode, compressMode,
1085 lastRevision, newRevision,
1086 excludePaths, excludePatterns)
1087 else:
1088 logger.debug("Mbox directory will not be backed up, per collect mode.")
1089 if collectMode == 'incr':
1090 _writeNewRevision(config, mboxDir, newRevision)
1091 logger.info("Executed the mbox extended action successfully.")
1092
1094 """
1095 Gets the collect mode that should be used for an mbox file or directory.
1096 Use file- or directory-specific value if possible, otherwise take from mbox section.
1097 @param local: LocalConfig object.
1098 @param item: Mbox file or directory
1099 @return: Collect mode to use.
1100 """
1101 if item.collectMode is None:
1102 collectMode = local.mbox.collectMode
1103 else:
1104 collectMode = item.collectMode
1105 logger.debug("Collect mode is [%s]", collectMode)
1106 return collectMode
1107
1109 """
1110 Gets the compress mode that should be used for an mbox file or directory.
1111 Use file- or directory-specific value if possible, otherwise take from mbox section.
1112 @param local: LocalConfig object.
1113 @param item: Mbox file or directory
1114 @return: Compress mode to use.
1115 """
1116 if item.compressMode is None:
1117 compressMode = local.mbox.compressMode
1118 else:
1119 compressMode = item.compressMode
1120 logger.debug("Compress mode is [%s]", compressMode)
1121 return compressMode
1122
1124 """
1125 Gets the path to the revision file associated with a repository.
1126 @param config: Cedar Backup configuration.
1127 @param item: Mbox file or directory
1128 @return: Absolute path to the revision file associated with the repository.
1129 """
1130 normalized = buildNormalizedPath(item.absolutePath)
1131 filename = "%s.%s" % (normalized, REVISION_PATH_EXTENSION)
1132 revisionPath = os.path.join(config.options.workingDir, filename)
1133 logger.debug("Revision file path is [%s]", revisionPath)
1134 return revisionPath
1135
1137 """
1138 Loads the last revision date for this item from disk and returns it.
1139
1140 If this is a full backup, or if the revision file cannot be loaded for some
1141 reason, then C{None} is returned. This indicates that there is no previous
1142 revision, so the entire mail file or directory should be backed up.
1143
1144 @note: We write the actual revision object to disk via pickle, so we don't
1145 deal with the datetime precision or format at all. Whatever's in the object
1146 is what we write.
1147
1148 @param config: Cedar Backup configuration.
1149 @param item: Mbox file or directory
1150 @param fullBackup: Indicates whether this is a full backup
1151 @param collectMode: Indicates the collect mode for this item
1152
1153 @return: Revision date as a datetime.datetime object or C{None}.
1154 """
1155 revisionPath = _getRevisionPath(config, item)
1156 if fullBackup:
1157 revisionDate = None
1158 logger.debug("Revision file ignored because this is a full backup.")
1159 elif collectMode in ['weekly', 'daily']:
1160 revisionDate = None
1161 logger.debug("No revision file based on collect mode [%s].", collectMode)
1162 else:
1163 logger.debug("Revision file will be used for non-full incremental backup.")
1164 if not os.path.isfile(revisionPath):
1165 revisionDate = None
1166 logger.debug("Revision file [%s] does not exist on disk.", revisionPath)
1167 else:
1168 try:
1169 revisionDate = pickle.load(open(revisionPath, "r"))
1170 logger.debug("Loaded revision file [%s] from disk: [%s]", revisionPath, revisionDate)
1171 except:
1172 revisionDate = None
1173 logger.error("Failed loading revision file [%s] from disk.", revisionPath)
1174 return revisionDate
1175
1177 """
1178 Writes new revision information to disk.
1179
1180 If we can't write the revision file successfully for any reason, we'll log
1181 the condition but won't throw an exception.
1182
1183 @note: We write the actual revision object to disk via pickle, so we don't
1184 deal with the datetime precision or format at all. Whatever's in the object
1185 is what we write.
1186
1187 @param config: Cedar Backup configuration.
1188 @param item: Mbox file or directory
1189 @param newRevision: Revision date as a datetime.datetime object.
1190 """
1191 revisionPath = _getRevisionPath(config, item)
1192 try:
1193 pickle.dump(newRevision, open(revisionPath, "w"))
1194 changeOwnership(revisionPath, config.options.backupUser, config.options.backupGroup)
1195 logger.debug("Wrote new revision file [%s] to disk: [%s]", revisionPath, newRevision)
1196 except:
1197 logger.error("Failed to write revision file [%s] to disk.", revisionPath)
1198
1200 """
1201 Gets exclusions (file and patterns) associated with an mbox directory.
1202
1203 The returned files value is a list of absolute paths to be excluded from the
1204 backup for a given directory. It is derived from the mbox directory's
1205 relative exclude paths.
1206
1207 The returned patterns value is a list of patterns to be excluded from the
1208 backup for a given directory. It is derived from the mbox directory's list
1209 of patterns.
1210
1211 @param mboxDir: Mbox directory object.
1212
1213 @return: Tuple (files, patterns) indicating what to exclude.
1214 """
1215 paths = []
1216 if mboxDir.relativeExcludePaths is not None:
1217 for relativePath in mboxDir.relativeExcludePaths:
1218 paths.append(os.path.join(mboxDir.absolutePath, relativePath))
1219 patterns = []
1220 if mboxDir.excludePatterns is not None:
1221 patterns.extend(mboxDir.excludePatterns)
1222 logger.debug("Exclude paths: %s", paths)
1223 logger.debug("Exclude patterns: %s", patterns)
1224 return(paths, patterns)
1225
1226 -def _getBackupPath(config, mboxPath, compressMode, newRevision, targetDir=None):
1227 """
1228 Gets the backup file path (including correct extension) associated with an mbox path.
1229
1230 We assume that if the target directory is passed in, that we're backing up a
1231 directory. Under these circumstances, we'll just use the basename of the
1232 individual path as the output file.
1233
1234 @note: The backup path only contains the current date in YYYYMMDD format,
1235 but that's OK because the index information (stored elsewhere) is the actual
1236 date object.
1237
1238 @param config: Cedar Backup configuration.
1239 @param mboxPath: Path to the indicated mbox file or directory
1240 @param compressMode: Compress mode to use for this mbox path
1241 @param newRevision: Revision this backup path represents
1242 @param targetDir: Target directory in which the path should exist
1243
1244 @return: Absolute path to the backup file associated with the repository.
1245 """
1246 if targetDir is None:
1247 normalizedPath = buildNormalizedPath(mboxPath)
1248 revisionDate = newRevision.strftime("%Y%m%d")
1249 filename = "mbox-%s-%s" % (revisionDate, normalizedPath)
1250 else:
1251 filename = os.path.basename(mboxPath)
1252 if compressMode == 'gzip':
1253 filename = "%s.gz" % filename
1254 elif compressMode == 'bzip2':
1255 filename = "%s.bz2" % filename
1256 if targetDir is None:
1257 backupPath = os.path.join(config.collect.targetDir, filename)
1258 else:
1259 backupPath = os.path.join(targetDir, filename)
1260 logger.debug("Backup file path is [%s]", backupPath)
1261 return backupPath
1262
1264 """
1265 Gets the tarfile backup file path (including correct extension) associated
1266 with an mbox path.
1267
1268 Along with the path, the tar archive mode is returned in a form that can
1269 be used with L{BackupFileList.generateTarfile}.
1270
1271 @note: The tarfile path only contains the current date in YYYYMMDD format,
1272 but that's OK because the index information (stored elsewhere) is the actual
1273 date object.
1274
1275 @param config: Cedar Backup configuration.
1276 @param mboxPath: Path to the indicated mbox file or directory
1277 @param compressMode: Compress mode to use for this mbox path
1278 @param newRevision: Revision this backup path represents
1279
1280 @return: Tuple of (absolute path to tarfile, tar archive mode)
1281 """
1282 normalizedPath = buildNormalizedPath(mboxPath)
1283 revisionDate = newRevision.strftime("%Y%m%d")
1284 filename = "mbox-%s-%s.tar" % (revisionDate, normalizedPath)
1285 if compressMode == 'gzip':
1286 filename = "%s.gz" % filename
1287 archiveMode = "targz"
1288 elif compressMode == 'bzip2':
1289 filename = "%s.bz2" % filename
1290 archiveMode = "tarbz2"
1291 else:
1292 archiveMode = "tar"
1293 tarfilePath = os.path.join(config.collect.targetDir, filename)
1294 logger.debug("Tarfile path is [%s]", tarfilePath)
1295 return (tarfilePath, archiveMode)
1296
1298 """
1299 Opens the output file used for saving backup information.
1300
1301 If the compress mode is "gzip", we'll open a C{GzipFile}, and if the
1302 compress mode is "bzip2", we'll open a C{BZ2File}. Otherwise, we'll just
1303 return an object from the normal C{open()} method.
1304
1305 @param backupPath: Path to file to open.
1306 @param compressMode: Compress mode of file ("none", "gzip", "bzip").
1307
1308 @return: Output file object.
1309 """
1310 if compressMode == "gzip":
1311 return GzipFile(backupPath, "w")
1312 elif compressMode == "bzip2":
1313 return BZ2File(backupPath, "w")
1314 else:
1315 return open(backupPath, "w")
1316
1317 -def _backupMboxFile(config, absolutePath,
1318 fullBackup, collectMode, compressMode,
1319 lastRevision, newRevision, targetDir=None):
1320 """
1321 Backs up an individual mbox file.
1322
1323 @param config: Cedar Backup configuration.
1324 @param absolutePath: Path to mbox file to back up.
1325 @param fullBackup: Indicates whether this should be a full backup.
1326 @param collectMode: Indicates the collect mode for this item
1327 @param compressMode: Compress mode of file ("none", "gzip", "bzip")
1328 @param lastRevision: Date of last backup as datetime.datetime
1329 @param newRevision: Date of new (current) backup as datetime.datetime
1330 @param targetDir: Target directory to write the backed-up file into
1331
1332 @raise ValueError: If some value is missing or invalid.
1333 @raise IOError: If there is a problem backing up the mbox file.
1334 """
1335 backupPath = _getBackupPath(config, absolutePath, compressMode, newRevision, targetDir=targetDir)
1336 outputFile = _getOutputFile(backupPath, compressMode)
1337 if fullBackup or collectMode != "incr" or lastRevision is None:
1338 args = [ "-a", "-u", absolutePath, ]
1339 else:
1340 revisionDate = lastRevision.strftime("%Y-%m-%dT%H:%M:%S")
1341 args = [ "-a", "-u", "-d", "since %s" % revisionDate, absolutePath, ]
1342 command = resolveCommand(GREPMAIL_COMMAND)
1343 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=outputFile)[0]
1344 if result != 0:
1345 raise IOError("Error [%d] executing grepmail on [%s]." % (result, absolutePath))
1346 logger.debug("Completed backing up mailbox [%s].", absolutePath)
1347 return backupPath
1348
1349 -def _backupMboxDir(config, absolutePath,
1350 fullBackup, collectMode, compressMode,
1351 lastRevision, newRevision,
1352 excludePaths, excludePatterns):
1353 """
1354 Backs up a directory containing mbox files.
1355
1356 @param config: Cedar Backup configuration.
1357 @param absolutePath: Path to mbox directory to back up.
1358 @param fullBackup: Indicates whether this should be a full backup.
1359 @param collectMode: Indicates the collect mode for this item
1360 @param compressMode: Compress mode of file ("none", "gzip", "bzip")
1361 @param lastRevision: Date of last backup as datetime.datetime
1362 @param newRevision: Date of new (current) backup as datetime.datetime
1363 @param excludePaths: List of absolute paths to exclude.
1364 @param excludePatterns: List of patterns to exclude.
1365
1366 @raise ValueError: If some value is missing or invalid.
1367 @raise IOError: If there is a problem backing up the mbox file.
1368 """
1369 try:
1370 tmpdir = tempfile.mkdtemp(dir=config.options.workingDir)
1371 mboxList = FilesystemList()
1372 mboxList.excludeDirs = True
1373 mboxList.excludePaths = excludePaths
1374 mboxList.excludePatterns = excludePatterns
1375 mboxList.addDirContents(absolutePath, recursive=False)
1376 tarList = BackupFileList()
1377 for item in mboxList:
1378 backupPath = _backupMboxFile(config, item, fullBackup,
1379 collectMode, "none",
1380 lastRevision, newRevision,
1381 targetDir=tmpdir)
1382 tarList.addFile(backupPath)
1383 (tarfilePath, archiveMode) = _getTarfilePath(config, absolutePath, compressMode, newRevision)
1384 tarList.generateTarfile(tarfilePath, archiveMode, ignore=True, flat=True)
1385 changeOwnership(tarfilePath, config.options.backupUser, config.options.backupGroup)
1386 logger.debug("Completed backing up directory [%s].", absolutePath)
1387 finally:
1388 try:
1389 for item in tarList:
1390 if os.path.exists(item):
1391 try:
1392 os.remove(item)
1393 except: pass
1394 except: pass
1395 try:
1396 os.rmdir(tmpdir)
1397 except: pass
1398