1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41 """
42 Provides an extension to back up PostgreSQL databases.
43
44 This is a Cedar Backup extension used to back up PostgreSQL databases via the
45 Cedar Backup command line. It requires a new configurations section
46 <postgresql> and is intended to be run either immediately before or immediately
47 after the standard collect action. Aside from its own configuration, it
48 requires the options and collect configuration sections in the standard Cedar
49 Backup configuration file.
50
51 The backup is done via the C{pg_dump} or C{pg_dumpall} commands included with
52 the PostgreSQL product. Output can be compressed using C{gzip} or C{bzip2}.
53 Administrators can configure the extension either to back up all databases or
54 to back up only specific databases. The extension assumes that the current
55 user has passwordless access to the database since there is no easy way to pass
56 a password to the C{pg_dump} client. This can be accomplished using appropriate
57 voodoo in the C{pg_hda.conf} file.
58
59 Note that this code always produces a full backup. There is currently no
60 facility for making incremental backups.
61
62 You should always make C{/etc/cback.conf} unreadble to non-root users once you
63 place postgresql configuration into it, since postgresql configuration will
64 contain information about available PostgreSQL databases and usernames.
65
66 Use of this extension I{may} expose usernames in the process listing (via
67 C{ps}) when the backup is running if the username is specified in the
68 configuration.
69
70 @author: Kenneth J. Pronovici <pronovic@ieee.org>
71 @author: Antoine Beaupre <anarcat@koumbit.org>
72 """
73
74
75
76
77
78
79 import os
80 import logging
81 from gzip import GzipFile
82 from bz2 import BZ2File
83
84
85 from CedarBackup2.xmlutil import createInputDom, addContainerNode, addStringNode, addBooleanNode
86 from CedarBackup2.xmlutil import readFirstChild, readString, readStringList, readBoolean
87 from CedarBackup2.config import VALID_COMPRESS_MODES
88 from CedarBackup2.util import resolveCommand, executeCommand
89 from CedarBackup2.util import ObjectTypeList, changeOwnership
90
91
92
93
94
95
96 logger = logging.getLogger("CedarBackup2.log.extend.postgresql")
97 POSTGRESQLDUMP_COMMAND = [ "pg_dump", ]
98 POSTGRESQLDUMPALL_COMMAND = [ "pg_dumpall", ]
99
100
101
102
103
104
105 -class PostgresqlConfig(object):
106
107 """
108 Class representing PostgreSQL configuration.
109
110 The PostgreSQL configuration information is used for backing up PostgreSQL databases.
111
112 The following restrictions exist on data in this class:
113
114 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
115 - The 'all' flag must be 'Y' if no databases are defined.
116 - The 'all' flag must be 'N' if any databases are defined.
117 - Any values in the databases list must be strings.
118
119 @sort: __init__, __repr__, __str__, __cmp__, user, all, databases
120 """
121
122 - def __init__(self, user=None, compressMode=None, all=None, databases=None):
123 """
124 Constructor for the C{PostgresqlConfig} class.
125
126 @param user: User to execute backup as.
127 @param compressMode: Compress mode for backed-up files.
128 @param all: Indicates whether to back up all databases.
129 @param databases: List of databases to back up.
130 """
131 self._user = None
132 self._compressMode = None
133 self._all = None
134 self._databases = None
135 self.user = user
136 self.compressMode = compressMode
137 self.all = all
138 self.databases = databases
139
140 - def __repr__(self):
141 """
142 Official string representation for class instance.
143 """
144 return "PostgresqlConfig(%s, %s, %s)" % (self.user, self.all, self.databases)
145
147 """
148 Informal string representation for class instance.
149 """
150 return self.__repr__()
151
152 - def __cmp__(self, other):
153 """
154 Definition of equals operator for this class.
155 @param other: Other object to compare to.
156 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
157 """
158 if other is None:
159 return 1
160 if self.user != other.user:
161 if self.user < other.user:
162 return -1
163 else:
164 return 1
165 if self.compressMode != other.compressMode:
166 if self.compressMode < other.compressMode:
167 return -1
168 else:
169 return 1
170 if self.all != other.all:
171 if self.all < other.all:
172 return -1
173 else:
174 return 1
175 if self.databases != other.databases:
176 if self.databases < other.databases:
177 return -1
178 else:
179 return 1
180 return 0
181
182 - def _setUser(self, value):
183 """
184 Property target used to set the user value.
185 """
186 if value is not None:
187 if len(value) < 1:
188 raise ValueError("User must be non-empty string.")
189 self._user = value
190
191 - def _getUser(self):
192 """
193 Property target used to get the user value.
194 """
195 return self._user
196
197 - def _setCompressMode(self, value):
198 """
199 Property target used to set the compress mode.
200 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
201 @raise ValueError: If the value is not valid.
202 """
203 if value is not None:
204 if value not in VALID_COMPRESS_MODES:
205 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
206 self._compressMode = value
207
209 """
210 Property target used to get the compress mode.
211 """
212 return self._compressMode
213
214 - def _setAll(self, value):
215 """
216 Property target used to set the 'all' flag.
217 No validations, but we normalize the value to C{True} or C{False}.
218 """
219 if value:
220 self._all = True
221 else:
222 self._all = False
223
225 """
226 Property target used to get the 'all' flag.
227 """
228 return self._all
229
230 - def _setDatabases(self, value):
231 """
232 Property target used to set the databases list.
233 Either the value must be C{None} or each element must be a string.
234 @raise ValueError: If the value is not a string.
235 """
236 if value is None:
237 self._databases = None
238 else:
239 for database in value:
240 if len(database) < 1:
241 raise ValueError("Each database must be a non-empty string.")
242 try:
243 saved = self._databases
244 self._databases = ObjectTypeList(basestring, "string")
245 self._databases.extend(value)
246 except Exception, e:
247 self._databases = saved
248 raise e
249
250 - def _getDatabases(self):
251 """
252 Property target used to get the databases list.
253 """
254 return self._databases
255
256 user = property(_getUser, _setUser, None, "User to execute backup as.")
257 compressMode = property(_getCompressMode, _setCompressMode, None, "Compress mode to be used for backed-up files.")
258 all = property(_getAll, _setAll, None, "Indicates whether to back up all databases.")
259 databases = property(_getDatabases, _setDatabases, None, "List of databases to back up.")
260
267
268 """
269 Class representing this extension's configuration document.
270
271 This is not a general-purpose configuration object like the main Cedar
272 Backup configuration object. Instead, it just knows how to parse and emit
273 PostgreSQL-specific configuration values. Third parties who need to read and
274 write configuration related to this extension should access it through the
275 constructor, C{validate} and C{addConfig} methods.
276
277 @note: Lists within this class are "unordered" for equality comparisons.
278
279 @sort: __init__, __repr__, __str__, __cmp__, postgresql, validate, addConfig
280 """
281
282 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
283 """
284 Initializes a configuration object.
285
286 If you initialize the object without passing either C{xmlData} or
287 C{xmlPath} then configuration will be empty and will be invalid until it
288 is filled in properly.
289
290 No reference to the original XML data or original path is saved off by
291 this class. Once the data has been parsed (successfully or not) this
292 original information is discarded.
293
294 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
295 method will be called (with its default arguments) against configuration
296 after successfully parsing any passed-in XML. Keep in mind that even if
297 C{validate} is C{False}, it might not be possible to parse the passed-in
298 XML document if lower-level validations fail.
299
300 @note: It is strongly suggested that the C{validate} option always be set
301 to C{True} (the default) unless there is a specific need to read in
302 invalid configuration from disk.
303
304 @param xmlData: XML data representing configuration.
305 @type xmlData: String data.
306
307 @param xmlPath: Path to an XML file on disk.
308 @type xmlPath: Absolute path to a file on disk.
309
310 @param validate: Validate the document after parsing it.
311 @type validate: Boolean true/false.
312
313 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
314 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
315 @raise ValueError: If the parsed configuration document is not valid.
316 """
317 self._postgresql = None
318 self.postgresql = None
319 if xmlData is not None and xmlPath is not None:
320 raise ValueError("Use either xmlData or xmlPath, but not both.")
321 if xmlData is not None:
322 self._parseXmlData(xmlData)
323 if validate:
324 self.validate()
325 elif xmlPath is not None:
326 xmlData = open(xmlPath).read()
327 self._parseXmlData(xmlData)
328 if validate:
329 self.validate()
330
332 """
333 Official string representation for class instance.
334 """
335 return "LocalConfig(%s)" % (self.postgresql)
336
338 """
339 Informal string representation for class instance.
340 """
341 return self.__repr__()
342
344 """
345 Definition of equals operator for this class.
346 Lists within this class are "unordered" for equality comparisons.
347 @param other: Other object to compare to.
348 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
349 """
350 if other is None:
351 return 1
352 if self.postgresql != other.postgresql:
353 if self.postgresql < other.postgresql:
354 return -1
355 else:
356 return 1
357 return 0
358
359 - def _setPostgresql(self, value):
360 """
361 Property target used to set the postgresql configuration value.
362 If not C{None}, the value must be a C{PostgresqlConfig} object.
363 @raise ValueError: If the value is not a C{PostgresqlConfig}
364 """
365 if value is None:
366 self._postgresql = None
367 else:
368 if not isinstance(value, PostgresqlConfig):
369 raise ValueError("Value must be a C{PostgresqlConfig} object.")
370 self._postgresql = value
371
372 - def _getPostgresql(self):
373 """
374 Property target used to get the postgresql configuration value.
375 """
376 return self._postgresql
377
378 postgresql = property(_getPostgresql, _setPostgresql, None, "Postgresql configuration in terms of a C{PostgresqlConfig} object.")
379
381 """
382 Validates configuration represented by the object.
383
384 The compress mode must be filled in. Then, if the 'all' flag
385 I{is} set, no databases are allowed, and if the 'all' flag is
386 I{not} set, at least one database is required.
387
388 @raise ValueError: If one of the validations fails.
389 """
390 if self.postgresql is None:
391 raise ValueError("PostgreSQL section is required.")
392 if self.postgresql.compressMode is None:
393 raise ValueError("Compress mode value is required.")
394 if self.postgresql.all:
395 if self.postgresql.databases is not None and self.postgresql.databases != []:
396 raise ValueError("Databases cannot be specified if 'all' flag is set.")
397 else:
398 if self.postgresql.databases is None or len(self.postgresql.databases) < 1:
399 raise ValueError("At least one PostgreSQL database must be indicated if 'all' flag is not set.")
400
402 """
403 Adds a <postgresql> configuration section as the next child of a parent.
404
405 Third parties should use this function to write configuration related to
406 this extension.
407
408 We add the following fields to the document::
409
410 user //cb_config/postgresql/user
411 compressMode //cb_config/postgresql/compress_mode
412 all //cb_config/postgresql/all
413
414 We also add groups of the following items, one list element per
415 item::
416
417 database //cb_config/postgresql/database
418
419 @param xmlDom: DOM tree as from C{impl.createDocument()}.
420 @param parentNode: Parent that the section should be appended to.
421 """
422 if self.postgresql is not None:
423 sectionNode = addContainerNode(xmlDom, parentNode, "postgresql")
424 addStringNode(xmlDom, sectionNode, "user", self.postgresql.user)
425 addStringNode(xmlDom, sectionNode, "compress_mode", self.postgresql.compressMode)
426 addBooleanNode(xmlDom, sectionNode, "all", self.postgresql.all)
427 if self.postgresql.databases is not None:
428 for database in self.postgresql.databases:
429 addStringNode(xmlDom, sectionNode, "database", database)
430
432 """
433 Internal method to parse an XML string into the object.
434
435 This method parses the XML document into a DOM tree (C{xmlDom}) and then
436 calls a static method to parse the postgresql configuration section.
437
438 @param xmlData: XML data to be parsed
439 @type xmlData: String data
440
441 @raise ValueError: If the XML cannot be successfully parsed.
442 """
443 (xmlDom, parentNode) = createInputDom(xmlData)
444 self._postgresql = LocalConfig._parsePostgresql(parentNode)
445
446 @staticmethod
447 - def _parsePostgresql(parent):
448 """
449 Parses a postgresql configuration section.
450
451 We read the following fields::
452
453 user //cb_config/postgresql/user
454 compressMode //cb_config/postgresql/compress_mode
455 all //cb_config/postgresql/all
456
457 We also read groups of the following item, one list element per
458 item::
459
460 databases //cb_config/postgresql/database
461
462 @param parent: Parent node to search beneath.
463
464 @return: C{PostgresqlConfig} object or C{None} if the section does not exist.
465 @raise ValueError: If some filled-in value is invalid.
466 """
467 postgresql = None
468 section = readFirstChild(parent, "postgresql")
469 if section is not None:
470 postgresql = PostgresqlConfig()
471 postgresql.user = readString(section, "user")
472 postgresql.compressMode = readString(section, "compress_mode")
473 postgresql.all = readBoolean(section, "all")
474 postgresql.databases = readStringList(section, "database")
475 return postgresql
476
477
478
479
480
481
482
483
484
485
486 -def executeAction(configPath, options, config):
487 """
488 Executes the PostgreSQL backup action.
489
490 @param configPath: Path to configuration file on disk.
491 @type configPath: String representing a path on disk.
492
493 @param options: Program command-line options.
494 @type options: Options object.
495
496 @param config: Program configuration.
497 @type config: Config object.
498
499 @raise ValueError: Under many generic error conditions
500 @raise IOError: If a backup could not be written for some reason.
501 """
502 logger.debug("Executing PostgreSQL extended action.")
503 if config.options is None or config.collect is None:
504 raise ValueError("Cedar Backup configuration is not properly filled in.")
505 local = LocalConfig(xmlPath=configPath)
506 if local.postgresql.all:
507 logger.info("Backing up all databases.")
508 _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user,
509 config.options.backupUser, config.options.backupGroup, None)
510 if local.postgresql.databases is not None and local.postgresql.databases != []:
511 logger.debug("Backing up %d individual databases." % len(local.postgresql.databases))
512 for database in local.postgresql.databases:
513 logger.info("Backing up database [%s]." % database)
514 _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user,
515 config.options.backupUser, config.options.backupGroup, database)
516 logger.info("Executed the PostgreSQL extended action successfully.")
517
518 -def _backupDatabase(targetDir, compressMode, user, backupUser, backupGroup, database=None):
519 """
520 Backs up an individual PostgreSQL database, or all databases.
521
522 This internal method wraps the public method and adds some functionality,
523 like figuring out a filename, etc.
524
525 @param targetDir: Directory into which backups should be written.
526 @param compressMode: Compress mode to be used for backed-up files.
527 @param user: User to use for connecting to the database.
528 @param backupUser: User to own resulting file.
529 @param backupGroup: Group to own resulting file.
530 @param database: Name of database, or C{None} for all databases.
531
532 @return: Name of the generated backup file.
533
534 @raise ValueError: If some value is missing or invalid.
535 @raise IOError: If there is a problem executing the PostgreSQL dump.
536 """
537 (outputFile, filename) = _getOutputFile(targetDir, database, compressMode)
538 try:
539 backupDatabase(user, outputFile, database)
540 finally:
541 outputFile.close()
542 if not os.path.exists(filename):
543 raise IOError("Dump file [%s] does not seem to exist after backup completed." % filename)
544 changeOwnership(filename, backupUser, backupGroup)
545
547 """
548 Opens the output file used for saving the PostgreSQL dump.
549
550 The filename is either C{"postgresqldump.txt"} or
551 C{"postgresqldump-<database>.txt"}. The C{".gz"} or C{".bz2"} extension is
552 added if C{compress} is C{True}.
553
554 @param targetDir: Target directory to write file in.
555 @param database: Name of the database (if any)
556 @param compressMode: Compress mode to be used for backed-up files.
557
558 @return: Tuple of (Output file object, filename)
559 """
560 if database is None:
561 filename = os.path.join(targetDir, "postgresqldump.txt")
562 else:
563 filename = os.path.join(targetDir, "postgresqldump-%s.txt" % database)
564 if compressMode == "gzip":
565 filename = "%s.gz" % filename
566 outputFile = GzipFile(filename, "w")
567 elif compressMode == "bzip2":
568 filename = "%s.bz2" % filename
569 outputFile = BZ2File(filename, "w")
570 else:
571 outputFile = open(filename, "w")
572 logger.debug("PostgreSQL dump file will be [%s]." % filename)
573 return (outputFile, filename)
574
575
576
577
578
579
580 -def backupDatabase(user, backupFile, database=None):
581 """
582 Backs up an individual PostgreSQL database, or all databases.
583
584 This function backs up either a named local PostgreSQL database or all local
585 PostgreSQL databases, using the passed in user for connectivity.
586 This is I{always} a full backup. There is no facility for incremental
587 backups.
588
589 The backup data will be written into the passed-in back file. Normally,
590 this would be an object as returned from C{open()}, but it is possible to
591 use something like a C{GzipFile} to write compressed output. The caller is
592 responsible for closing the passed-in backup file.
593
594 @note: Typically, you would use the C{root} user to back up all databases.
595
596 @param user: User to use for connecting to the database.
597 @type user: String representing PostgreSQL username.
598
599 @param backupFile: File use for writing backup.
600 @type backupFile: Python file object as from C{open()} or C{file()}.
601
602 @param database: Name of the database to be backed up.
603 @type database: String representing database name, or C{None} for all databases.
604
605 @raise ValueError: If some value is missing or invalid.
606 @raise IOError: If there is a problem executing the PostgreSQL dump.
607 """
608 args = []
609 if user is not None:
610 args.append('-U')
611 args.append(user)
612
613 if database is None:
614 command = resolveCommand(POSTGRESQLDUMPALL_COMMAND)
615 else:
616 command = resolveCommand(POSTGRESQLDUMP_COMMAND)
617 args.append(database)
618
619 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=backupFile)[0]
620 if result != 0:
621 if database is None:
622 raise IOError("Error [%d] executing PostgreSQL database dump for all databases." % result)
623 else:
624 raise IOError("Error [%d] executing PostgreSQL database dump for database [%s]." % (result, database))
625