1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 """
39 Provides an extension to back up Subversion repositories.
40
41 This is a Cedar Backup extension used to back up Subversion repositories via
42 the Cedar Backup command line. Each Subversion repository can be backed using
43 the same collect modes allowed for filesystems in the standard Cedar Backup
44 collect action: weekly, daily, incremental.
45
46 This extension requires a new configuration section <subversion> and is
47 intended to be run either immediately before or immediately after the standard
48 collect action. Aside from its own configuration, it requires the options and
49 collect configuration sections in the standard Cedar Backup configuration file.
50
51 There are two different kinds of Subversion repositories at this writing: BDB
52 (Berkeley Database) and FSFS (a "filesystem within a filesystem"). Although
53 the repository type can be specified in configuration, that information is just
54 kept around for reference. It doesn't affect the backup. Both kinds of
55 repositories are backed up in the same way, using C{svnadmin dump} in an
56 incremental mode.
57
58 It turns out that FSFS repositories can also be backed up just like any
59 other filesystem directory. If you would rather do that, then use the normal
60 collect action. This is probably simpler, although it carries its own
61 advantages and disadvantages (plus you will have to be careful to exclude
62 the working directories Subversion uses when building an update to commit).
63 Check the Subversion documentation for more information.
64
65 @author: Kenneth J. Pronovici <pronovic@ieee.org>
66 """
67
68
69
70
71
72
73 import os
74 import logging
75 import pickle
76 from bz2 import BZ2File
77 from gzip import GzipFile
78
79
80 from CedarBackup2.xmlutil import createInputDom, addContainerNode, addStringNode
81 from CedarBackup2.xmlutil import isElement, readChildren, readFirstChild, readString, readStringList
82 from CedarBackup2.config import VALID_COLLECT_MODES, VALID_COMPRESS_MODES
83 from CedarBackup2.filesystem import FilesystemList
84 from CedarBackup2.util import UnorderedList, RegexList
85 from CedarBackup2.util import isStartOfWeek, buildNormalizedPath
86 from CedarBackup2.util import resolveCommand, executeCommand
87 from CedarBackup2.util import ObjectTypeList, encodePath, changeOwnership
88
89
90
91
92
93
94 logger = logging.getLogger("CedarBackup2.log.extend.subversion")
95
96 SVNLOOK_COMMAND = [ "svnlook", ]
97 SVNADMIN_COMMAND = [ "svnadmin", ]
98
99 REVISION_PATH_EXTENSION = "svnlast"
107
108 """
109 Class representing Subversion repository directory.
110
111 A repository directory is a directory that contains one or more Subversion
112 repositories.
113
114 The following restrictions exist on data in this class:
115
116 - The directory path must be absolute.
117 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
118 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
119
120 The repository type value is kept around just for reference. It doesn't
121 affect the behavior of the backup.
122
123 Relative exclusions are allowed here. However, there is no configured
124 ignore file, because repository dir backups are not recursive.
125
126 @sort: __init__, __repr__, __str__, __cmp__, directoryPath, collectMode, compressMode
127 """
128
129 - def __init__(self, repositoryType=None, directoryPath=None, collectMode=None, compressMode=None,
130 relativeExcludePaths=None, excludePatterns=None):
131 """
132 Constructor for the C{RepositoryDir} class.
133
134 @param repositoryType: Type of repository, for reference
135 @param directoryPath: Absolute path of the Subversion parent directory
136 @param collectMode: Overridden collect mode for this directory.
137 @param compressMode: Overridden compression mode for this directory.
138 @param relativeExcludePaths: List of relative paths to exclude.
139 @param excludePatterns: List of regular expression patterns to exclude
140 """
141 self._repositoryType = None
142 self._directoryPath = None
143 self._collectMode = None
144 self._compressMode = None
145 self._relativeExcludePaths = None
146 self._excludePatterns = None
147 self.repositoryType = repositoryType
148 self.directoryPath = directoryPath
149 self.collectMode = collectMode
150 self.compressMode = compressMode
151 self.relativeExcludePaths = relativeExcludePaths
152 self.excludePatterns = excludePatterns
153
155 """
156 Official string representation for class instance.
157 """
158 return "RepositoryDir(%s, %s, %s, %s, %s, %s)" % (self.repositoryType, self.directoryPath, self.collectMode,
159 self.compressMode, self.relativeExcludePaths, self.excludePatterns)
160
162 """
163 Informal string representation for class instance.
164 """
165 return self.__repr__()
166
206
208 """
209 Property target used to set the repository type.
210 There is no validation; this value is kept around just for reference.
211 """
212 self._repositoryType = value
213
215 """
216 Property target used to get the repository type.
217 """
218 return self._repositoryType
219
221 """
222 Property target used to set the directory path.
223 The value must be an absolute path if it is not C{None}.
224 It does not have to exist on disk at the time of assignment.
225 @raise ValueError: If the value is not an absolute path.
226 @raise ValueError: If the value cannot be encoded properly.
227 """
228 if value is not None:
229 if not os.path.isabs(value):
230 raise ValueError("Repository path must be an absolute path.")
231 self._directoryPath = encodePath(value)
232
234 """
235 Property target used to get the repository path.
236 """
237 return self._directoryPath
238
240 """
241 Property target used to set the collect mode.
242 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
243 @raise ValueError: If the value is not valid.
244 """
245 if value is not None:
246 if value not in VALID_COLLECT_MODES:
247 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
248 self._collectMode = value
249
251 """
252 Property target used to get the collect mode.
253 """
254 return self._collectMode
255
257 """
258 Property target used to set the compress mode.
259 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
260 @raise ValueError: If the value is not valid.
261 """
262 if value is not None:
263 if value not in VALID_COMPRESS_MODES:
264 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
265 self._compressMode = value
266
268 """
269 Property target used to get the compress mode.
270 """
271 return self._compressMode
272
274 """
275 Property target used to set the relative exclude paths list.
276 Elements do not have to exist on disk at the time of assignment.
277 """
278 if value is None:
279 self._relativeExcludePaths = None
280 else:
281 try:
282 saved = self._relativeExcludePaths
283 self._relativeExcludePaths = UnorderedList()
284 self._relativeExcludePaths.extend(value)
285 except Exception, e:
286 self._relativeExcludePaths = saved
287 raise e
288
290 """
291 Property target used to get the relative exclude paths list.
292 """
293 return self._relativeExcludePaths
294
296 """
297 Property target used to set the exclude patterns list.
298 """
299 if value is None:
300 self._excludePatterns = None
301 else:
302 try:
303 saved = self._excludePatterns
304 self._excludePatterns = RegexList()
305 self._excludePatterns.extend(value)
306 except Exception, e:
307 self._excludePatterns = saved
308 raise e
309
311 """
312 Property target used to get the exclude patterns list.
313 """
314 return self._excludePatterns
315
316 repositoryType = property(_getRepositoryType, _setRepositoryType, None, doc="Type of this repository, for reference.")
317 directoryPath = property(_getDirectoryPath, _setDirectoryPath, None, doc="Absolute path of the Subversion parent directory.")
318 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this repository.")
319 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this repository.")
320 relativeExcludePaths = property(_getRelativeExcludePaths, _setRelativeExcludePaths, None, "List of relative paths to exclude.")
321 excludePatterns = property(_getExcludePatterns, _setExcludePatterns, None, "List of regular expression patterns to exclude.")
322
329
330 """
331 Class representing generic Subversion repository configuration..
332
333 The following restrictions exist on data in this class:
334
335 - The respository path must be absolute.
336 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
337 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
338
339 The repository type value is kept around just for reference. It doesn't
340 affect the behavior of the backup.
341
342 @sort: __init__, __repr__, __str__, __cmp__, repositoryPath, collectMode, compressMode
343 """
344
345 - def __init__(self, repositoryType=None, repositoryPath=None, collectMode=None, compressMode=None):
346 """
347 Constructor for the C{Repository} class.
348
349 @param repositoryType: Type of repository, for reference
350 @param repositoryPath: Absolute path to a Subversion repository on disk.
351 @param collectMode: Overridden collect mode for this directory.
352 @param compressMode: Overridden compression mode for this directory.
353 """
354 self._repositoryType = None
355 self._repositoryPath = None
356 self._collectMode = None
357 self._compressMode = None
358 self.repositoryType = repositoryType
359 self.repositoryPath = repositoryPath
360 self.collectMode = collectMode
361 self.compressMode = compressMode
362
368
370 """
371 Informal string representation for class instance.
372 """
373 return self.__repr__()
374
404
406 """
407 Property target used to set the repository type.
408 There is no validation; this value is kept around just for reference.
409 """
410 self._repositoryType = value
411
413 """
414 Property target used to get the repository type.
415 """
416 return self._repositoryType
417
419 """
420 Property target used to set the repository path.
421 The value must be an absolute path if it is not C{None}.
422 It does not have to exist on disk at the time of assignment.
423 @raise ValueError: If the value is not an absolute path.
424 @raise ValueError: If the value cannot be encoded properly.
425 """
426 if value is not None:
427 if not os.path.isabs(value):
428 raise ValueError("Repository path must be an absolute path.")
429 self._repositoryPath = encodePath(value)
430
432 """
433 Property target used to get the repository path.
434 """
435 return self._repositoryPath
436
438 """
439 Property target used to set the collect mode.
440 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
441 @raise ValueError: If the value is not valid.
442 """
443 if value is not None:
444 if value not in VALID_COLLECT_MODES:
445 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
446 self._collectMode = value
447
449 """
450 Property target used to get the collect mode.
451 """
452 return self._collectMode
453
455 """
456 Property target used to set the compress mode.
457 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
458 @raise ValueError: If the value is not valid.
459 """
460 if value is not None:
461 if value not in VALID_COMPRESS_MODES:
462 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
463 self._compressMode = value
464
466 """
467 Property target used to get the compress mode.
468 """
469 return self._compressMode
470
471 repositoryType = property(_getRepositoryType, _setRepositoryType, None, doc="Type of this repository, for reference.")
472 repositoryPath = property(_getRepositoryPath, _setRepositoryPath, None, doc="Path to the repository to collect.")
473 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this repository.")
474 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this repository.")
475
482
483 """
484 Class representing Subversion configuration.
485
486 Subversion configuration is used for backing up Subversion repositories.
487
488 The following restrictions exist on data in this class:
489
490 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
491 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
492 - The repositories list must be a list of C{Repository} objects.
493 - The repositoryDirs list must be a list of C{RepositoryDir} objects.
494
495 For the two lists, validation is accomplished through the
496 L{util.ObjectTypeList} list implementation that overrides common list
497 methods and transparently ensures that each element has the correct type.
498
499 @note: Lists within this class are "unordered" for equality comparisons.
500
501 @sort: __init__, __repr__, __str__, __cmp__, collectMode, compressMode, repositories
502 """
503
504 - def __init__(self, collectMode=None, compressMode=None, repositories=None, repositoryDirs=None):
505 """
506 Constructor for the C{SubversionConfig} class.
507
508 @param collectMode: Default collect mode.
509 @param compressMode: Default compress mode.
510 @param repositories: List of Subversion repositories to back up.
511 @param repositoryDirs: List of Subversion parent directories to back up.
512
513 @raise ValueError: If one of the values is invalid.
514 """
515 self._collectMode = None
516 self._compressMode = None
517 self._repositories = None
518 self._repositoryDirs = None
519 self.collectMode = collectMode
520 self.compressMode = compressMode
521 self.repositories = repositories
522 self.repositoryDirs = repositoryDirs
523
529
531 """
532 Informal string representation for class instance.
533 """
534 return self.__repr__()
535
566
568 """
569 Property target used to set the collect mode.
570 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
571 @raise ValueError: If the value is not valid.
572 """
573 if value is not None:
574 if value not in VALID_COLLECT_MODES:
575 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
576 self._collectMode = value
577
579 """
580 Property target used to get the collect mode.
581 """
582 return self._collectMode
583
585 """
586 Property target used to set the compress mode.
587 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
588 @raise ValueError: If the value is not valid.
589 """
590 if value is not None:
591 if value not in VALID_COMPRESS_MODES:
592 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
593 self._compressMode = value
594
596 """
597 Property target used to get the compress mode.
598 """
599 return self._compressMode
600
602 """
603 Property target used to set the repositories list.
604 Either the value must be C{None} or each element must be a C{Repository}.
605 @raise ValueError: If the value is not a C{Repository}
606 """
607 if value is None:
608 self._repositories = None
609 else:
610 try:
611 saved = self._repositories
612 self._repositories = ObjectTypeList(Repository, "Repository")
613 self._repositories.extend(value)
614 except Exception, e:
615 self._repositories = saved
616 raise e
617
619 """
620 Property target used to get the repositories list.
621 """
622 return self._repositories
623
625 """
626 Property target used to set the repositoryDirs list.
627 Either the value must be C{None} or each element must be a C{Repository}.
628 @raise ValueError: If the value is not a C{Repository}
629 """
630 if value is None:
631 self._repositoryDirs = None
632 else:
633 try:
634 saved = self._repositoryDirs
635 self._repositoryDirs = ObjectTypeList(RepositoryDir, "RepositoryDir")
636 self._repositoryDirs.extend(value)
637 except Exception, e:
638 self._repositoryDirs = saved
639 raise e
640
642 """
643 Property target used to get the repositoryDirs list.
644 """
645 return self._repositoryDirs
646
647 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Default collect mode.")
648 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Default compress mode.")
649 repositories = property(_getRepositories, _setRepositories, None, doc="List of Subversion repositories to back up.")
650 repositoryDirs = property(_getRepositoryDirs, _setRepositoryDirs, None, doc="List of Subversion parent directories to back up.")
651
658
659 """
660 Class representing this extension's configuration document.
661
662 This is not a general-purpose configuration object like the main Cedar
663 Backup configuration object. Instead, it just knows how to parse and emit
664 Subversion-specific configuration values. Third parties who need to read
665 and write configuration related to this extension should access it through
666 the constructor, C{validate} and C{addConfig} methods.
667
668 @note: Lists within this class are "unordered" for equality comparisons.
669
670 @sort: __init__, __repr__, __str__, __cmp__, subversion, validate, addConfig
671 """
672
673 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
674 """
675 Initializes a configuration object.
676
677 If you initialize the object without passing either C{xmlData} or
678 C{xmlPath} then configuration will be empty and will be invalid until it
679 is filled in properly.
680
681 No reference to the original XML data or original path is saved off by
682 this class. Once the data has been parsed (successfully or not) this
683 original information is discarded.
684
685 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
686 method will be called (with its default arguments) against configuration
687 after successfully parsing any passed-in XML. Keep in mind that even if
688 C{validate} is C{False}, it might not be possible to parse the passed-in
689 XML document if lower-level validations fail.
690
691 @note: It is strongly suggested that the C{validate} option always be set
692 to C{True} (the default) unless there is a specific need to read in
693 invalid configuration from disk.
694
695 @param xmlData: XML data representing configuration.
696 @type xmlData: String data.
697
698 @param xmlPath: Path to an XML file on disk.
699 @type xmlPath: Absolute path to a file on disk.
700
701 @param validate: Validate the document after parsing it.
702 @type validate: Boolean true/false.
703
704 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
705 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
706 @raise ValueError: If the parsed configuration document is not valid.
707 """
708 self._subversion = None
709 self.subversion = None
710 if xmlData is not None and xmlPath is not None:
711 raise ValueError("Use either xmlData or xmlPath, but not both.")
712 if xmlData is not None:
713 self._parseXmlData(xmlData)
714 if validate:
715 self.validate()
716 elif xmlPath is not None:
717 xmlData = open(xmlPath).read()
718 self._parseXmlData(xmlData)
719 if validate:
720 self.validate()
721
723 """
724 Official string representation for class instance.
725 """
726 return "LocalConfig(%s)" % (self.subversion)
727
729 """
730 Informal string representation for class instance.
731 """
732 return self.__repr__()
733
735 """
736 Definition of equals operator for this class.
737 Lists within this class are "unordered" for equality comparisons.
738 @param other: Other object to compare to.
739 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
740 """
741 if other is None:
742 return 1
743 if self.subversion != other.subversion:
744 if self.subversion < other.subversion:
745 return -1
746 else:
747 return 1
748 return 0
749
751 """
752 Property target used to set the subversion configuration value.
753 If not C{None}, the value must be a C{SubversionConfig} object.
754 @raise ValueError: If the value is not a C{SubversionConfig}
755 """
756 if value is None:
757 self._subversion = None
758 else:
759 if not isinstance(value, SubversionConfig):
760 raise ValueError("Value must be a C{SubversionConfig} object.")
761 self._subversion = value
762
764 """
765 Property target used to get the subversion configuration value.
766 """
767 return self._subversion
768
769 subversion = property(_getSubversion, _setSubversion, None, "Subversion configuration in terms of a C{SubversionConfig} object.")
770
772 """
773 Validates configuration represented by the object.
774
775 Subversion configuration must be filled in. Within that, the collect
776 mode and compress mode are both optional, but the list of repositories
777 must contain at least one entry.
778
779 Each repository must contain a repository path, and then must be either
780 able to take collect mode and compress mode configuration from the parent
781 C{SubversionConfig} object, or must set each value on its own.
782
783 @raise ValueError: If one of the validations fails.
784 """
785 if self.subversion is None:
786 raise ValueError("Subversion section is required.")
787 if ((self.subversion.repositories is None or len(self.subversion.repositories) < 1) and
788 (self.subversion.repositoryDirs is None or len(self.subversion.repositoryDirs) <1)):
789 raise ValueError("At least one Subversion repository must be configured.")
790 if self.subversion.repositories is not None:
791 for repository in self.subversion.repositories:
792 if repository.repositoryPath is None:
793 raise ValueError("Each repository must set a repository path.")
794 if self.subversion.collectMode is None and repository.collectMode is None:
795 raise ValueError("Collect mode must either be set in parent section or individual repository.")
796 if self.subversion.compressMode is None and repository.compressMode is None:
797 raise ValueError("Compress mode must either be set in parent section or individual repository.")
798 if self.subversion.repositoryDirs is not None:
799 for repositoryDir in self.subversion.repositoryDirs:
800 if repositoryDir.directoryPath is None:
801 raise ValueError("Each repository directory must set a directory path.")
802 if self.subversion.collectMode is None and repositoryDir.collectMode is None:
803 raise ValueError("Collect mode must either be set in parent section or repository directory.")
804 if self.subversion.compressMode is None and repositoryDir.compressMode is None:
805 raise ValueError("Compress mode must either be set in parent section or repository directory.")
806
808 """
809 Adds a <subversion> configuration section as the next child of a parent.
810
811 Third parties should use this function to write configuration related to
812 this extension.
813
814 We add the following fields to the document::
815
816 collectMode //cb_config/subversion/collectMode
817 compressMode //cb_config/subversion/compressMode
818
819 We also add groups of the following items, one list element per
820 item::
821
822 repository //cb_config/subversion/repository
823 repository_dir //cb_config/subversion/repository_dir
824
825 @param xmlDom: DOM tree as from C{impl.createDocument()}.
826 @param parentNode: Parent that the section should be appended to.
827 """
828 if self.subversion is not None:
829 sectionNode = addContainerNode(xmlDom, parentNode, "subversion")
830 addStringNode(xmlDom, sectionNode, "collect_mode", self.subversion.collectMode)
831 addStringNode(xmlDom, sectionNode, "compress_mode", self.subversion.compressMode)
832 if self.subversion.repositories is not None:
833 for repository in self.subversion.repositories:
834 LocalConfig._addRepository(xmlDom, sectionNode, repository)
835 if self.subversion.repositoryDirs is not None:
836 for repositoryDir in self.subversion.repositoryDirs:
837 LocalConfig._addRepositoryDir(xmlDom, sectionNode, repositoryDir)
838
840 """
841 Internal method to parse an XML string into the object.
842
843 This method parses the XML document into a DOM tree (C{xmlDom}) and then
844 calls a static method to parse the subversion configuration section.
845
846 @param xmlData: XML data to be parsed
847 @type xmlData: String data
848
849 @raise ValueError: If the XML cannot be successfully parsed.
850 """
851 (xmlDom, parentNode) = createInputDom(xmlData)
852 self._subversion = LocalConfig._parseSubversion(parentNode)
853
854 @staticmethod
856 """
857 Parses a subversion configuration section.
858
859 We read the following individual fields::
860
861 collectMode //cb_config/subversion/collect_mode
862 compressMode //cb_config/subversion/compress_mode
863
864 We also read groups of the following item, one list element per
865 item::
866
867 repositories //cb_config/subversion/repository
868 repository_dirs //cb_config/subversion/repository_dir
869
870 The repositories are parsed by L{_parseRepositories}, and the repository
871 dirs are parsed by L{_parseRepositoryDirs}.
872
873 @param parent: Parent node to search beneath.
874
875 @return: C{SubversionConfig} object or C{None} if the section does not exist.
876 @raise ValueError: If some filled-in value is invalid.
877 """
878 subversion = None
879 section = readFirstChild(parent, "subversion")
880 if section is not None:
881 subversion = SubversionConfig()
882 subversion.collectMode = readString(section, "collect_mode")
883 subversion.compressMode = readString(section, "compress_mode")
884 subversion.repositories = LocalConfig._parseRepositories(section)
885 subversion.repositoryDirs = LocalConfig._parseRepositoryDirs(section)
886 return subversion
887
888 @staticmethod
890 """
891 Reads a list of C{Repository} objects from immediately beneath the parent.
892
893 We read the following individual fields::
894
895 repositoryType type
896 repositoryPath abs_path
897 collectMode collect_mode
898 compressMode compess_mode
899
900 The type field is optional, and its value is kept around only for
901 reference.
902
903 @param parent: Parent node to search beneath.
904
905 @return: List of C{Repository} objects or C{None} if none are found.
906 @raise ValueError: If some filled-in value is invalid.
907 """
908 lst = []
909 for entry in readChildren(parent, "repository"):
910 if isElement(entry):
911 repository = Repository()
912 repository.repositoryType = readString(entry, "type")
913 repository.repositoryPath = readString(entry, "abs_path")
914 repository.collectMode = readString(entry, "collect_mode")
915 repository.compressMode = readString(entry, "compress_mode")
916 lst.append(repository)
917 if lst == []:
918 lst = None
919 return lst
920
921 @staticmethod
923 """
924 Adds a repository container as the next child of a parent.
925
926 We add the following fields to the document::
927
928 repositoryType repository/type
929 repositoryPath repository/abs_path
930 collectMode repository/collect_mode
931 compressMode repository/compress_mode
932
933 The <repository> node itself is created as the next child of the parent
934 node. This method only adds one repository node. The parent must loop
935 for each repository in the C{SubversionConfig} object.
936
937 If C{repository} is C{None}, this method call will be a no-op.
938
939 @param xmlDom: DOM tree as from C{impl.createDocument()}.
940 @param parentNode: Parent that the section should be appended to.
941 @param repository: Repository to be added to the document.
942 """
943 if repository is not None:
944 sectionNode = addContainerNode(xmlDom, parentNode, "repository")
945 addStringNode(xmlDom, sectionNode, "type", repository.repositoryType)
946 addStringNode(xmlDom, sectionNode, "abs_path", repository.repositoryPath)
947 addStringNode(xmlDom, sectionNode, "collect_mode", repository.collectMode)
948 addStringNode(xmlDom, sectionNode, "compress_mode", repository.compressMode)
949
950 @staticmethod
952 """
953 Reads a list of C{RepositoryDir} objects from immediately beneath the parent.
954
955 We read the following individual fields::
956
957 repositoryType type
958 directoryPath abs_path
959 collectMode collect_mode
960 compressMode compess_mode
961
962 We also read groups of the following items, one list element per
963 item::
964
965 relativeExcludePaths exclude/rel_path
966 excludePatterns exclude/pattern
967
968 The exclusions are parsed by L{_parseExclusions}.
969
970 The type field is optional, and its value is kept around only for
971 reference.
972
973 @param parent: Parent node to search beneath.
974
975 @return: List of C{RepositoryDir} objects or C{None} if none are found.
976 @raise ValueError: If some filled-in value is invalid.
977 """
978 lst = []
979 for entry in readChildren(parent, "repository_dir"):
980 if isElement(entry):
981 repositoryDir = RepositoryDir()
982 repositoryDir.repositoryType = readString(entry, "type")
983 repositoryDir.directoryPath = readString(entry, "abs_path")
984 repositoryDir.collectMode = readString(entry, "collect_mode")
985 repositoryDir.compressMode = readString(entry, "compress_mode")
986 (repositoryDir.relativeExcludePaths, repositoryDir.excludePatterns) = LocalConfig._parseExclusions(entry)
987 lst.append(repositoryDir)
988 if lst == []:
989 lst = None
990 return lst
991
992 @staticmethod
994 """
995 Reads exclusions data from immediately beneath the parent.
996
997 We read groups of the following items, one list element per item::
998
999 relative exclude/rel_path
1000 patterns exclude/pattern
1001
1002 If there are none of some pattern (i.e. no relative path items) then
1003 C{None} will be returned for that item in the tuple.
1004
1005 @param parentNode: Parent node to search beneath.
1006
1007 @return: Tuple of (relative, patterns) exclusions.
1008 """
1009 section = readFirstChild(parentNode, "exclude")
1010 if section is None:
1011 return (None, None)
1012 else:
1013 relative = readStringList(section, "rel_path")
1014 patterns = readStringList(section, "pattern")
1015 return (relative, patterns)
1016
1017 @staticmethod
1019 """
1020 Adds a repository dir container as the next child of a parent.
1021
1022 We add the following fields to the document::
1023
1024 repositoryType repository_dir/type
1025 directoryPath repository_dir/abs_path
1026 collectMode repository_dir/collect_mode
1027 compressMode repository_dir/compress_mode
1028
1029 We also add groups of the following items, one list element per item::
1030
1031 relativeExcludePaths dir/exclude/rel_path
1032 excludePatterns dir/exclude/pattern
1033
1034 The <repository_dir> node itself is created as the next child of the
1035 parent node. This method only adds one repository node. The parent must
1036 loop for each repository dir in the C{SubversionConfig} object.
1037
1038 If C{repositoryDir} is C{None}, this method call will be a no-op.
1039
1040 @param xmlDom: DOM tree as from C{impl.createDocument()}.
1041 @param parentNode: Parent that the section should be appended to.
1042 @param repositoryDir: Repository dir to be added to the document.
1043 """
1044 if repositoryDir is not None:
1045 sectionNode = addContainerNode(xmlDom, parentNode, "repository_dir")
1046 addStringNode(xmlDom, sectionNode, "type", repositoryDir.repositoryType)
1047 addStringNode(xmlDom, sectionNode, "abs_path", repositoryDir.directoryPath)
1048 addStringNode(xmlDom, sectionNode, "collect_mode", repositoryDir.collectMode)
1049 addStringNode(xmlDom, sectionNode, "compress_mode", repositoryDir.compressMode)
1050 if ((repositoryDir.relativeExcludePaths is not None and repositoryDir.relativeExcludePaths != []) or
1051 (repositoryDir.excludePatterns is not None and repositoryDir.excludePatterns != [])):
1052 excludeNode = addContainerNode(xmlDom, sectionNode, "exclude")
1053 if repositoryDir.relativeExcludePaths is not None:
1054 for relativePath in repositoryDir.relativeExcludePaths:
1055 addStringNode(xmlDom, excludeNode, "rel_path", relativePath)
1056 if repositoryDir.excludePatterns is not None:
1057 for pattern in repositoryDir.excludePatterns:
1058 addStringNode(xmlDom, excludeNode, "pattern", pattern)
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069 -def executeAction(configPath, options, config):
1070 """
1071 Executes the Subversion backup action.
1072
1073 @param configPath: Path to configuration file on disk.
1074 @type configPath: String representing a path on disk.
1075
1076 @param options: Program command-line options.
1077 @type options: Options object.
1078
1079 @param config: Program configuration.
1080 @type config: Config object.
1081
1082 @raise ValueError: Under many generic error conditions
1083 @raise IOError: If a backup could not be written for some reason.
1084 """
1085 logger.debug("Executing Subversion extended action.")
1086 if config.options is None or config.collect is None:
1087 raise ValueError("Cedar Backup configuration is not properly filled in.")
1088 local = LocalConfig(xmlPath=configPath)
1089 todayIsStart = isStartOfWeek(config.options.startingDay)
1090 fullBackup = options.full or todayIsStart
1091 logger.debug("Full backup flag is [%s]" % fullBackup)
1092 if local.subversion.repositories is not None:
1093 for repository in local.subversion.repositories:
1094 _backupRepository(config, local, todayIsStart, fullBackup, repository)
1095 if local.subversion.repositoryDirs is not None:
1096 for repositoryDir in local.subversion.repositoryDirs:
1097 logger.debug("Working with repository directory [%s]." % repositoryDir.directoryPath)
1098 for repositoryPath in _getRepositoryPaths(repositoryDir):
1099 repository = Repository(repositoryDir.repositoryType, repositoryPath,
1100 repositoryDir.collectMode, repositoryDir.compressMode)
1101 _backupRepository(config, local, todayIsStart, fullBackup, repository)
1102 logger.info("Completed backing up Subversion repository directory [%s]." % repositoryDir.directoryPath)
1103 logger.info("Executed the Subversion extended action successfully.")
1104
1118
1133
1135 """
1136 Gets the path to the revision file associated with a repository.
1137 @param config: Config object.
1138 @param repository: Repository object.
1139 @return: Absolute path to the revision file associated with the repository.
1140 """
1141 normalized = buildNormalizedPath(repository.repositoryPath)
1142 filename = "%s.%s" % (normalized, REVISION_PATH_EXTENSION)
1143 revisionPath = os.path.join(config.options.workingDir, filename)
1144 logger.debug("Revision file path is [%s]" % revisionPath)
1145 return revisionPath
1146
1147 -def _getBackupPath(config, repositoryPath, compressMode, startRevision, endRevision):
1148 """
1149 Gets the backup file path (including correct extension) associated with a repository.
1150 @param config: Config object.
1151 @param repositoryPath: Path to the indicated repository
1152 @param compressMode: Compress mode to use for this repository.
1153 @param startRevision: Starting repository revision.
1154 @param endRevision: Ending repository revision.
1155 @return: Absolute path to the backup file associated with the repository.
1156 """
1157 normalizedPath = buildNormalizedPath(repositoryPath)
1158 filename = "svndump-%d:%d-%s.txt" % (startRevision, endRevision, normalizedPath)
1159 if compressMode == 'gzip':
1160 filename = "%s.gz" % filename
1161 elif compressMode == 'bzip2':
1162 filename = "%s.bz2" % filename
1163 backupPath = os.path.join(config.collect.targetDir, filename)
1164 logger.debug("Backup file path is [%s]" % backupPath)
1165 return backupPath
1166
1180
1182 """
1183 Gets exclusions (file and patterns) associated with an repository directory.
1184
1185 The returned files value is a list of absolute paths to be excluded from the
1186 backup for a given directory. It is derived from the repository directory's
1187 relative exclude paths.
1188
1189 The returned patterns value is a list of patterns to be excluded from the
1190 backup for a given directory. It is derived from the repository directory's
1191 list of patterns.
1192
1193 @param repositoryDir: Repository directory object.
1194
1195 @return: Tuple (files, patterns) indicating what to exclude.
1196 """
1197 paths = []
1198 if repositoryDir.relativeExcludePaths is not None:
1199 for relativePath in repositoryDir.relativeExcludePaths:
1200 paths.append(os.path.join(repositoryDir.directoryPath, relativePath))
1201 patterns = []
1202 if repositoryDir.excludePatterns is not None:
1203 patterns.extend(repositoryDir.excludePatterns)
1204 logger.debug("Exclude paths: %s" % paths)
1205 logger.debug("Exclude patterns: %s" % patterns)
1206 return(paths, patterns)
1207
1209 """
1210 Backs up an individual Subversion repository.
1211
1212 This internal method wraps the public methods and adds some functionality
1213 to work better with the extended action itself.
1214
1215 @param config: Cedar Backup configuration.
1216 @param local: Local configuration
1217 @param todayIsStart: Indicates whether today is start of week
1218 @param fullBackup: Full backup flag
1219 @param repository: Repository to operate on
1220
1221 @raise ValueError: If some value is missing or invalid.
1222 @raise IOError: If there is a problem executing the Subversion dump.
1223 """
1224 logger.debug("Working with repository [%s]" % repository.repositoryPath)
1225 logger.debug("Repository type is [%s]" % repository.repositoryType)
1226 collectMode = _getCollectMode(local, repository)
1227 compressMode = _getCompressMode(local, repository)
1228 revisionPath = _getRevisionPath(config, repository)
1229 if not (fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart)):
1230 logger.debug("Repository will not be backed up, per collect mode.")
1231 return
1232 logger.debug("Repository meets criteria to be backed up today.")
1233 if collectMode != "incr" or fullBackup:
1234 startRevision = 0
1235 endRevision = getYoungestRevision(repository.repositoryPath)
1236 logger.debug("Using full backup, revision: (%d, %d)." % (startRevision, endRevision))
1237 else:
1238 if fullBackup:
1239 startRevision = 0
1240 endRevision = getYoungestRevision(repository.repositoryPath)
1241 else:
1242 startRevision = _loadLastRevision(revisionPath) + 1
1243 endRevision = getYoungestRevision(repository.repositoryPath)
1244 if startRevision > endRevision:
1245 logger.info("No need to back up repository [%s]; no new revisions." % repository.repositoryPath)
1246 return
1247 logger.debug("Using incremental backup, revision: (%d, %d)." % (startRevision, endRevision))
1248 backupPath = _getBackupPath(config, repository.repositoryPath, compressMode, startRevision, endRevision)
1249 outputFile = _getOutputFile(backupPath, compressMode)
1250 try:
1251 backupRepository(repository.repositoryPath, outputFile, startRevision, endRevision)
1252 finally:
1253 outputFile.close()
1254 if not os.path.exists(backupPath):
1255 raise IOError("Dump file [%s] does not seem to exist after backup completed." % backupPath)
1256 changeOwnership(backupPath, config.options.backupUser, config.options.backupGroup)
1257 if collectMode == "incr":
1258 _writeLastRevision(config, revisionPath, endRevision)
1259 logger.info("Completed backing up Subversion repository [%s]." % repository.repositoryPath)
1260
1262 """
1263 Opens the output file used for saving the Subversion dump.
1264
1265 If the compress mode is "gzip", we'll open a C{GzipFile}, and if the
1266 compress mode is "bzip2", we'll open a C{BZ2File}. Otherwise, we'll just
1267 return an object from the normal C{open()} method.
1268
1269 @param backupPath: Path to file to open.
1270 @param compressMode: Compress mode of file ("none", "gzip", "bzip").
1271
1272 @return: Output file object.
1273 """
1274 if compressMode == "gzip":
1275 return GzipFile(backupPath, "w")
1276 elif compressMode == "bzip2":
1277 return BZ2File(backupPath, "w")
1278 else:
1279 return open(backupPath, "w")
1280
1282 """
1283 Loads the indicated revision file from disk into an integer.
1284
1285 If we can't load the revision file successfully (either because it doesn't
1286 exist or for some other reason), then a revision of -1 will be returned -
1287 but the condition will be logged. This way, we err on the side of backing
1288 up too much, because anyone using this will presumably be adding 1 to the
1289 revision, so they don't duplicate any backups.
1290
1291 @param revisionPath: Path to the revision file on disk.
1292
1293 @return: Integer representing last backed-up revision, -1 on error or if none can be read.
1294 """
1295 if not os.path.isfile(revisionPath):
1296 startRevision = -1
1297 logger.debug("Revision file [%s] does not exist on disk." % revisionPath)
1298 else:
1299 try:
1300 startRevision = pickle.load(open(revisionPath, "r"))
1301 logger.debug("Loaded revision file [%s] from disk: %d." % (revisionPath, startRevision))
1302 except:
1303 startRevision = -1
1304 logger.error("Failed loading revision file [%s] from disk." % revisionPath)
1305 return startRevision
1306
1308 """
1309 Writes the end revision to the indicated revision file on disk.
1310
1311 If we can't write the revision file successfully for any reason, we'll log
1312 the condition but won't throw an exception.
1313
1314 @param config: Config object.
1315 @param revisionPath: Path to the revision file on disk.
1316 @param endRevision: Last revision backed up on this run.
1317 """
1318 try:
1319 pickle.dump(endRevision, open(revisionPath, "w"))
1320 changeOwnership(revisionPath, config.options.backupUser, config.options.backupGroup)
1321 logger.debug("Wrote new revision file [%s] to disk: %d." % (revisionPath, endRevision))
1322 except:
1323 logger.error("Failed to write revision file [%s] to disk." % revisionPath)
1324
1325
1326
1327
1328
1329
1330 -def backupRepository(repositoryPath, backupFile, startRevision=None, endRevision=None):
1331 """
1332 Backs up an individual Subversion repository.
1333
1334 The starting and ending revision values control an incremental backup. If
1335 the starting revision is not passed in, then revision zero (the start of the
1336 repository) is assumed. If the ending revision is not passed in, then the
1337 youngest revision in the database will be used as the endpoint.
1338
1339 The backup data will be written into the passed-in back file. Normally,
1340 this would be an object as returned from C{open}, but it is possible to use
1341 something like a C{GzipFile} to write compressed output. The caller is
1342 responsible for closing the passed-in backup file.
1343
1344 @note: This function should either be run as root or as the owner of the
1345 Subversion repository.
1346
1347 @note: It is apparently I{not} a good idea to interrupt this function.
1348 Sometimes, this leaves the repository in a "wedged" state, which requires
1349 recovery using C{svnadmin recover}.
1350
1351 @param repositoryPath: Path to Subversion repository to back up
1352 @type repositoryPath: String path representing Subversion repository on disk.
1353
1354 @param backupFile: Python file object to use for writing backup.
1355 @type backupFile: Python file object as from C{open()} or C{file()}.
1356
1357 @param startRevision: Starting repository revision to back up (for incremental backups)
1358 @type startRevision: Integer value >= 0.
1359
1360 @param endRevision: Ending repository revision to back up (for incremental backups)
1361 @type endRevision: Integer value >= 0.
1362
1363 @raise ValueError: If some value is missing or invalid.
1364 @raise IOError: If there is a problem executing the Subversion dump.
1365 """
1366 if startRevision is None:
1367 startRevision = 0
1368 if endRevision is None:
1369 endRevision = getYoungestRevision(repositoryPath)
1370 if int(startRevision) < 0:
1371 raise ValueError("Start revision must be >= 0.")
1372 if int(endRevision) < 0:
1373 raise ValueError("End revision must be >= 0.")
1374 if startRevision > endRevision:
1375 raise ValueError("Start revision must be <= end revision.")
1376 args = [ "dump", "--quiet", "-r%s:%s" % (startRevision, endRevision), "--incremental", repositoryPath, ]
1377 command = resolveCommand(SVNADMIN_COMMAND)
1378 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=backupFile)[0]
1379 if result != 0:
1380 raise IOError("Error [%d] executing Subversion dump for repository [%s]." % (result, repositoryPath))
1381 logger.debug("Completed dumping subversion repository [%s]." % repositoryPath)
1382
1389 """
1390 Gets the youngest (newest) revision in a Subversion repository using C{svnlook}.
1391
1392 @note: This function should either be run as root or as the owner of the
1393 Subversion repository.
1394
1395 @param repositoryPath: Path to Subversion repository to look in.
1396 @type repositoryPath: String path representing Subversion repository on disk.
1397
1398 @return: Youngest revision as an integer.
1399
1400 @raise ValueError: If there is a problem parsing the C{svnlook} output.
1401 @raise IOError: If there is a problem executing the C{svnlook} command.
1402 """
1403 args = [ 'youngest', repositoryPath, ]
1404 command = resolveCommand(SVNLOOK_COMMAND)
1405 (result, output) = executeCommand(command, args, returnOutput=True, ignoreStderr=True)
1406 if result != 0:
1407 raise IOError("Error [%d] executing 'svnlook youngest' for repository [%s]." % (result, repositoryPath))
1408 if len(output) != 1:
1409 raise ValueError("Unable to parse 'svnlook youngest' output.")
1410 return int(output[0])
1411
1418
1419 """
1420 Class representing Subversion BDB (Berkeley Database) repository configuration.
1421 This object is deprecated. Use a simple L{Repository} instead.
1422 """
1423
1424 - def __init__(self, repositoryPath=None, collectMode=None, compressMode=None):
1429
1435
1438
1439 """
1440 Class representing Subversion FSFS repository configuration.
1441 This object is deprecated. Use a simple L{Repository} instead.
1442 """
1443
1444 - def __init__(self, repositoryPath=None, collectMode=None, compressMode=None):
1449
1455
1456
1457 -def backupBDBRepository(repositoryPath, backupFile, startRevision=None, endRevision=None):
1458 """
1459 Backs up an individual Subversion BDB repository.
1460 This function is deprecated. Use L{backupRepository} instead.
1461 """
1462 return backupRepository(repositoryPath, backupFile, startRevision, endRevision)
1463
1466 """
1467 Backs up an individual Subversion FSFS repository.
1468 This function is deprecated. Use L{backupRepository} instead.
1469 """
1470 return backupRepository(repositoryPath, backupFile, startRevision, endRevision)
1471