1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42 """
43 Provides general-purpose utilities.
44
45 @sort: AbsolutePathList, ObjectTypeList, RestrictedValueList, RegexMatchList,
46 RegexList, _Vertex, DirectedGraph, PathResolverSingleton,
47 sortDict, convertSize, getUidGid, changeOwnership, splitCommandLine,
48 resolveCommand, executeCommand, calculateFileAge, encodePath, nullDevice,
49 deriveDayOfWeek, isStartOfWeek, buildNormalizedPath,
50 ISO_SECTOR_SIZE, BYTES_PER_SECTOR,
51 BYTES_PER_KBYTE, BYTES_PER_MBYTE, BYTES_PER_GBYTE, KBYTES_PER_MBYTE, MBYTES_PER_GBYTE,
52 SECONDS_PER_MINUTE, MINUTES_PER_HOUR, HOURS_PER_DAY, SECONDS_PER_DAY,
53 UNIT_BYTES, UNIT_KBYTES, UNIT_MBYTES, UNIT_GBYTES, UNIT_SECTORS
54
55 @var ISO_SECTOR_SIZE: Size of an ISO image sector, in bytes.
56 @var BYTES_PER_SECTOR: Number of bytes (B) per ISO sector.
57 @var BYTES_PER_KBYTE: Number of bytes (B) per kilobyte (kB).
58 @var BYTES_PER_MBYTE: Number of bytes (B) per megabyte (MB).
59 @var BYTES_PER_GBYTE: Number of bytes (B) per megabyte (GB).
60 @var KBYTES_PER_MBYTE: Number of kilobytes (kB) per megabyte (MB).
61 @var MBYTES_PER_GBYTE: Number of megabytes (MB) per gigabyte (GB).
62 @var SECONDS_PER_MINUTE: Number of seconds per minute.
63 @var MINUTES_PER_HOUR: Number of minutes per hour.
64 @var HOURS_PER_DAY: Number of hours per day.
65 @var SECONDS_PER_DAY: Number of seconds per day.
66 @var UNIT_BYTES: Constant representing the byte (B) unit for conversion.
67 @var UNIT_KBYTES: Constant representing the kilobyte (kB) unit for conversion.
68 @var UNIT_MBYTES: Constant representing the megabyte (MB) unit for conversion.
69 @var UNIT_GBYTES: Constant representing the gigabyte (GB) unit for conversion.
70 @var UNIT_SECTORS: Constant representing the ISO sector unit for conversion.
71
72 @author: Kenneth J. Pronovici <pronovic@ieee.org>
73 """
74
75
76
77
78
79
80 import sys
81 import math
82 import os
83 import re
84 import time
85 import logging
86 import string
87
88 try:
89 import pwd
90 import grp
91 _UID_GID_AVAILABLE = True
92 except ImportError:
93 _UID_GID_AVAILABLE = False
94
95 try:
96 from subprocess import Popen
97 _PIPE_IMPLEMENTATION = "subprocess.Popen"
98 except ImportError:
99 try:
100 from popen2 import Popen4
101 _PIPE_IMPLEMENTATION = "popen2.Popen4"
102 except ImportError:
103 raise ImportError("Unable to import either subprocess.Popen or popen2.Popen4 for use by Pipe class.")
104
105
106
107
108
109
110 logger = logging.getLogger("CedarBackup2.log.util")
111 outputLogger = logging.getLogger("CedarBackup2.output")
112
113 ISO_SECTOR_SIZE = 2048.0
114 BYTES_PER_SECTOR = ISO_SECTOR_SIZE
115
116 BYTES_PER_KBYTE = 1024.0
117 KBYTES_PER_MBYTE = 1024.0
118 MBYTES_PER_GBYTE = 1024.0
119 BYTES_PER_MBYTE = BYTES_PER_KBYTE * KBYTES_PER_MBYTE
120 BYTES_PER_GBYTE = BYTES_PER_MBYTE * MBYTES_PER_GBYTE
121
122 SECONDS_PER_MINUTE = 60
123 MINUTES_PER_HOUR = 60
124 HOURS_PER_DAY = 24
125 SECONDS_PER_DAY = SECONDS_PER_MINUTE * MINUTES_PER_HOUR * HOURS_PER_DAY
126
127 UNIT_BYTES = 0
128 UNIT_KBYTES = 1
129 UNIT_MBYTES = 2
130 UNIT_GBYTES = 4
131 UNIT_SECTORS = 3
132
133 MTAB_FILE = "/etc/mtab"
134
135 MOUNT_COMMAND = [ "mount", ]
136 UMOUNT_COMMAND = [ "umount", ]
137
138
139
140
141
142
144
145 """
146 Class representing an "unordered list".
147
148 An "unordered list" is a list in which only the contents matter, not the
149 order in which the contents appear in the list.
150
151 For instance, we might be keeping track of set of paths in a list, because
152 it's convenient to have them in that form. However, for comparison
153 purposes, we would only care that the lists contain exactly the same
154 contents, regardless of order.
155
156 I have come up with two reasonable ways of doing this, plus a couple more
157 that would work but would be a pain to implement. My first method is to
158 copy and sort each list, comparing the sorted versions. This will only work
159 if two lists with exactly the same members are guaranteed to sort in exactly
160 the same order. The second way would be to create two Sets and then compare
161 the sets. However, this would lose information about any duplicates in
162 either list. I've decided to go with option #1 for now. I'll modify this
163 code if I run into problems in the future.
164
165 We override the original C{__eq__}, C{__ne__}, C{__ge__}, C{__gt__},
166 C{__le__} and C{__lt__} list methods to change the definition of the various
167 comparison operators. In all cases, the comparison is changed to return the
168 result of the original operation I{but instead comparing sorted lists}.
169 This is going to be quite a bit slower than a normal list, so you probably
170 only want to use it on small lists.
171 """
172
174 """
175 Definition of C{==} operator for this class.
176 @param other: Other object to compare to.
177 @return: True/false depending on whether C{self == other}.
178 """
179 if other is None:
180 return False
181 selfSorted = self[:]
182 otherSorted = other[:]
183 selfSorted.sort()
184 otherSorted.sort()
185 return selfSorted.__eq__(otherSorted)
186
188 """
189 Definition of C{!=} operator for this class.
190 @param other: Other object to compare to.
191 @return: True/false depending on whether C{self != other}.
192 """
193 if other is None:
194 return True
195 selfSorted = self[:]
196 otherSorted = other[:]
197 selfSorted.sort()
198 otherSorted.sort()
199 return selfSorted.__ne__(otherSorted)
200
202 """
203 Definition of S{>=} operator for this class.
204 @param other: Other object to compare to.
205 @return: True/false depending on whether C{self >= other}.
206 """
207 if other is None:
208 return True
209 selfSorted = self[:]
210 otherSorted = other[:]
211 selfSorted.sort()
212 otherSorted.sort()
213 return selfSorted.__ge__(otherSorted)
214
216 """
217 Definition of C{>} operator for this class.
218 @param other: Other object to compare to.
219 @return: True/false depending on whether C{self > other}.
220 """
221 if other is None:
222 return True
223 selfSorted = self[:]
224 otherSorted = other[:]
225 selfSorted.sort()
226 otherSorted.sort()
227 return selfSorted.__gt__(otherSorted)
228
230 """
231 Definition of S{<=} operator for this class.
232 @param other: Other object to compare to.
233 @return: True/false depending on whether C{self <= other}.
234 """
235 if other is None:
236 return False
237 selfSorted = self[:]
238 otherSorted = other[:]
239 selfSorted.sort()
240 otherSorted.sort()
241 return selfSorted.__le__(otherSorted)
242
244 """
245 Definition of C{<} operator for this class.
246 @param other: Other object to compare to.
247 @return: True/false depending on whether C{self < other}.
248 """
249 if other is None:
250 return False
251 selfSorted = self[:]
252 otherSorted = other[:]
253 selfSorted.sort()
254 otherSorted.sort()
255 return selfSorted.__lt__(otherSorted)
256
257
258
259
260
261
263
264 """
265 Class representing a list of absolute paths.
266
267 This is an unordered list.
268
269 We override the C{append}, C{insert} and C{extend} methods to ensure that
270 any item added to the list is an absolute path.
271
272 Each item added to the list is encoded using L{encodePath}. If we don't do
273 this, we have problems trying certain operations between strings and unicode
274 objects, particularly for "odd" filenames that can't be encoded in standard
275 ASCII.
276 """
277
279 """
280 Overrides the standard C{append} method.
281 @raise ValueError: If item is not an absolute path.
282 """
283 if not os.path.isabs(item):
284 raise ValueError("Not an absolute path: [%s]" % item)
285 list.append(self, encodePath(item))
286
287 - def insert(self, index, item):
288 """
289 Overrides the standard C{insert} method.
290 @raise ValueError: If item is not an absolute path.
291 """
292 if not os.path.isabs(item):
293 raise ValueError("Not an absolute path: [%s]" % item)
294 list.insert(self, index, encodePath(item))
295
297 """
298 Overrides the standard C{insert} method.
299 @raise ValueError: If any item is not an absolute path.
300 """
301 for item in seq:
302 if not os.path.isabs(item):
303 raise ValueError("Not an absolute path: [%s]" % item)
304 for item in seq:
305 list.append(self, encodePath(item))
306
307
308
309
310
311
313
314 """
315 Class representing a list containing only objects with a certain type.
316
317 This is an unordered list.
318
319 We override the C{append}, C{insert} and C{extend} methods to ensure that
320 any item added to the list matches the type that is requested. The
321 comparison uses the built-in C{isinstance}, which should allow subclasses of
322 of the requested type to be added to the list as well.
323
324 The C{objectName} value will be used in exceptions, i.e. C{"Item must be a
325 CollectDir object."} if C{objectName} is C{"CollectDir"}.
326 """
327
328 - def __init__(self, objectType, objectName):
329 """
330 Initializes a typed list for a particular type.
331 @param objectType: Type that the list elements must match.
332 @param objectName: Short string containing the "name" of the type.
333 """
334 self.objectType = objectType
335 self.objectName = objectName
336
338 """
339 Overrides the standard C{append} method.
340 @raise ValueError: If item does not match requested type.
341 """
342 if not isinstance(item, self.objectType):
343 raise ValueError("Item must be a %s object." % self.objectName)
344 list.append(self, item)
345
346 - def insert(self, index, item):
347 """
348 Overrides the standard C{insert} method.
349 @raise ValueError: If item does not match requested type.
350 """
351 if not isinstance(item, self.objectType):
352 raise ValueError("Item must be a %s object." % self.objectName)
353 list.insert(self, index, item)
354
356 """
357 Overrides the standard C{insert} method.
358 @raise ValueError: If item does not match requested type.
359 """
360 for item in seq:
361 if not isinstance(item, self.objectType):
362 raise ValueError("All items must be %s objects." % self.objectName)
363 list.extend(self, seq)
364
365
366
367
368
369
370 -class RestrictedContentList(UnorderedList):
371
372 """
373 Class representing a list containing only object with certain values.
374
375 This is an unordered list.
376
377 We override the C{append}, C{insert} and C{extend} methods to ensure that
378 any item added to the list is among the valid values. We use a standard
379 comparison, so pretty much anything can be in the list of valid values.
380
381 The C{valuesDescr} value will be used in exceptions, i.e. C{"Item must be
382 one of values in VALID_ACTIONS"} if C{valuesDescr} is C{"VALID_ACTIONS"}.
383
384 @note: This class doesn't make any attempt to trap for nonsensical
385 arguments. All of the values in the values list should be of the same type
386 (i.e. strings). Then, all list operations also need to be of that type
387 (i.e. you should always insert or append just strings). If you mix types --
388 for instance lists and strings -- you will likely see AttributeError
389 exceptions or other problems.
390 """
391
392 - def __init__(self, valuesList, valuesDescr, prefix=None):
393 """
394 Initializes a list restricted to containing certain values.
395 @param valuesList: List of valid values.
396 @param valuesDescr: Short string describing list of values.
397 @param prefix: Prefix to use in error messages (None results in prefix "Item")
398 """
399 self.prefix = "Item"
400 if prefix is not None: self.prefix = prefix
401 self.valuesList = valuesList
402 self.valuesDescr = valuesDescr
403
404 - def append(self, item):
405 """
406 Overrides the standard C{append} method.
407 @raise ValueError: If item is not in the values list.
408 """
409 if item not in self.valuesList:
410 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr))
411 list.append(self, item)
412
413 - def insert(self, index, item):
414 """
415 Overrides the standard C{insert} method.
416 @raise ValueError: If item is not in the values list.
417 """
418 if item not in self.valuesList:
419 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr))
420 list.insert(self, index, item)
421
422 - def extend(self, seq):
423 """
424 Overrides the standard C{insert} method.
425 @raise ValueError: If item is not in the values list.
426 """
427 for item in seq:
428 if item not in self.valuesList:
429 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr))
430 list.extend(self, seq)
431
432
433
434
435
436
438
439 """
440 Class representing a list containing only strings that match a regular expression.
441
442 If C{emptyAllowed} is passed in as C{False}, then empty strings are
443 explicitly disallowed, even if they happen to match the regular expression.
444 (C{None} values are always disallowed, since string operations are not
445 permitted on C{None}.)
446
447 This is an unordered list.
448
449 We override the C{append}, C{insert} and C{extend} methods to ensure that
450 any item added to the list matches the indicated regular expression.
451
452 @note: If you try to put values that are not strings into the list, you will
453 likely get either TypeError or AttributeError exceptions as a result.
454 """
455
456 - def __init__(self, valuesRegex, emptyAllowed=True, prefix=None):
457 """
458 Initializes a list restricted to containing certain values.
459 @param valuesRegex: Regular expression that must be matched, as a string
460 @param emptyAllowed: Indicates whether empty or None values are allowed.
461 @param prefix: Prefix to use in error messages (None results in prefix "Item")
462 """
463 self.prefix = "Item"
464 if prefix is not None: self.prefix = prefix
465 self.valuesRegex = valuesRegex
466 self.emptyAllowed = emptyAllowed
467 self.pattern = re.compile(self.valuesRegex)
468
470 """
471 Overrides the standard C{append} method.
472 @raise ValueError: If item is None
473 @raise ValueError: If item is empty and empty values are not allowed
474 @raise ValueError: If item does not match the configured regular expression
475 """
476 if item is None or (not self.emptyAllowed and item == ""):
477 raise ValueError("%s cannot be empty." % self.prefix)
478 if not self.pattern.search(item):
479 raise ValueError("%s is not valid: [%s]" % (self.prefix, item))
480 list.append(self, item)
481
482 - def insert(self, index, item):
483 """
484 Overrides the standard C{insert} method.
485 @raise ValueError: If item is None
486 @raise ValueError: If item is empty and empty values are not allowed
487 @raise ValueError: If item does not match the configured regular expression
488 """
489 if item is None or (not self.emptyAllowed and item == ""):
490 raise ValueError("%s cannot be empty." % self.prefix)
491 if not self.pattern.search(item):
492 raise ValueError("%s is not valid [%s]" % (self.prefix, item))
493 list.insert(self, index, item)
494
496 """
497 Overrides the standard C{insert} method.
498 @raise ValueError: If any item is None
499 @raise ValueError: If any item is empty and empty values are not allowed
500 @raise ValueError: If any item does not match the configured regular expression
501 """
502 for item in seq:
503 if item is None or (not self.emptyAllowed and item == ""):
504 raise ValueError("%s cannot be empty.", self.prefix)
505 if not self.pattern.search(item):
506 raise ValueError("%s is not valid: [%s]" % (self.prefix, item))
507 list.extend(self, seq)
508
509
510
511
512
513
515
516 """
517 Class representing a list of valid regular expression strings.
518
519 This is an unordered list.
520
521 We override the C{append}, C{insert} and C{extend} methods to ensure that
522 any item added to the list is a valid regular expression.
523 """
524
526 """
527 Overrides the standard C{append} method.
528 @raise ValueError: If item is not an absolute path.
529 """
530 try:
531 re.compile(item)
532 except re.error:
533 raise ValueError("Not a valid regular expression: [%s]" % item)
534 list.append(self, item)
535
536 - def insert(self, index, item):
537 """
538 Overrides the standard C{insert} method.
539 @raise ValueError: If item is not an absolute path.
540 """
541 try:
542 re.compile(item)
543 except re.error:
544 raise ValueError("Not a valid regular expression: [%s]" % item)
545 list.insert(self, index, item)
546
548 """
549 Overrides the standard C{insert} method.
550 @raise ValueError: If any item is not an absolute path.
551 """
552 for item in seq:
553 try:
554 re.compile(item)
555 except re.error:
556 raise ValueError("Not a valid regular expression: [%s]" % item)
557 for item in seq:
558 list.append(self, item)
559
560
561
562
563
564
566
567 """
568 Represents a vertex (or node) in a directed graph.
569 """
570
572 """
573 Constructor.
574 @param name: Name of this graph vertex.
575 @type name: String value.
576 """
577 self.name = name
578 self.endpoints = []
579 self.state = None
580
582
583 """
584 Represents a directed graph.
585
586 A graph B{G=(V,E)} consists of a set of vertices B{V} together with a set
587 B{E} of vertex pairs or edges. In a directed graph, each edge also has an
588 associated direction (from vertext B{v1} to vertex B{v2}). A C{DirectedGraph}
589 object provides a way to construct a directed graph and execute a depth-
590 first search.
591
592 This data structure was designed based on the graphing chapter in
593 U{The Algorithm Design Manual<http://www2.toki.or.id/book/AlgDesignManual/>},
594 by Steven S. Skiena.
595
596 This class is intended to be used by Cedar Backup for dependency ordering.
597 Because of this, it's not quite general-purpose. Unlike a "general" graph,
598 every vertex in this graph has at least one edge pointing to it, from a
599 special "start" vertex. This is so no vertices get "lost" either because
600 they have no dependencies or because nothing depends on them.
601 """
602
603 _UNDISCOVERED = 0
604 _DISCOVERED = 1
605 _EXPLORED = 2
606
608 """
609 Directed graph constructor.
610
611 @param name: Name of this graph.
612 @type name: String value.
613 """
614 if name is None or name == "":
615 raise ValueError("Graph name must be non-empty.")
616 self._name = name
617 self._vertices = {}
618 self._startVertex = _Vertex(None)
619
621 """
622 Official string representation for class instance.
623 """
624 return "DirectedGraph(%s)" % self.name
625
627 """
628 Informal string representation for class instance.
629 """
630 return self.__repr__()
631
633 """
634 Definition of equals operator for this class.
635 @param other: Other object to compare to.
636 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
637 """
638 if other is None:
639 return 1
640 if self._name != other._name:
641 if self._name < other._name:
642 return -1
643 else:
644 return 1
645 if self._vertices != other._vertices:
646 if self._vertices < other._vertices:
647 return -1
648 else:
649 return 1
650 return 0
651
653 """
654 Property target used to get the graph name.
655 """
656 return self._name
657
658 name = property(_getName, None, None, "Name of the graph.")
659
661 """
662 Creates a named vertex.
663 @param name: vertex name
664 @raise ValueError: If the vertex name is C{None} or empty.
665 """
666 if name is None or name == "":
667 raise ValueError("Vertex name must be non-empty.")
668 vertex = _Vertex(name)
669 self._startVertex.endpoints.append(vertex)
670 self._vertices[name] = vertex
671
673 """
674 Adds an edge with an associated direction, from C{start} vertex to C{finish} vertex.
675 @param start: Name of start vertex.
676 @param finish: Name of finish vertex.
677 @raise ValueError: If one of the named vertices is unknown.
678 """
679 try:
680 startVertex = self._vertices[start]
681 finishVertex = self._vertices[finish]
682 startVertex.endpoints.append(finishVertex)
683 except KeyError, e:
684 raise ValueError("Vertex [%s] could not be found." % e)
685
687 """
688 Implements a topological sort of the graph.
689
690 This method also enforces that the graph is a directed acyclic graph,
691 which is a requirement of a topological sort.
692
693 A directed acyclic graph (or "DAG") is a directed graph with no directed
694 cycles. A topological sort of a DAG is an ordering on the vertices such
695 that all edges go from left to right. Only an acyclic graph can have a
696 topological sort, but any DAG has at least one topological sort.
697
698 Since a topological sort only makes sense for an acyclic graph, this
699 method throws an exception if a cycle is found.
700
701 A depth-first search only makes sense if the graph is acyclic. If the
702 graph contains any cycles, it is not possible to determine a consistent
703 ordering for the vertices.
704
705 @note: If a particular vertex has no edges, then its position in the
706 final list depends on the order in which the vertices were created in the
707 graph. If you're using this method to determine a dependency order, this
708 makes sense: a vertex with no dependencies can go anywhere (and will).
709
710 @return: Ordering on the vertices so that all edges go from left to right.
711
712 @raise ValueError: If a cycle is found in the graph.
713 """
714 ordering = []
715 for key in self._vertices:
716 vertex = self._vertices[key]
717 vertex.state = self._UNDISCOVERED
718 for key in self._vertices:
719 vertex = self._vertices[key]
720 if vertex.state == self._UNDISCOVERED:
721 self._topologicalSort(self._startVertex, ordering)
722 return ordering
723
725 """
726 Recursive depth first search function implementing topological sort.
727 @param vertex: Vertex to search
728 @param ordering: List of vertices in proper order
729 """
730 vertex.state = self._DISCOVERED
731 for endpoint in vertex.endpoints:
732 if endpoint.state == self._UNDISCOVERED:
733 self._topologicalSort(endpoint, ordering)
734 elif endpoint.state != self._EXPLORED:
735 raise ValueError("Cycle found in graph (found '%s' while searching '%s')." % (vertex.name, endpoint.name))
736 if vertex.name is not None:
737 ordering.insert(0, vertex.name)
738 vertex.state = self._EXPLORED
739
740
741
742
743
744
746
747 """
748 Singleton used for resolving executable paths.
749
750 Various functions throughout Cedar Backup (including extensions) need a way
751 to resolve the path of executables that they use. For instance, the image
752 functionality needs to find the C{mkisofs} executable, and the Subversion
753 extension needs to find the C{svnlook} executable. Cedar Backup's original
754 behavior was to assume that the simple name (C{"svnlook"} or whatever) was
755 available on the caller's C{$PATH}, and to fail otherwise. However, this
756 turns out to be less than ideal, since for instance the root user might not
757 always have executables like C{svnlook} in its path.
758
759 One solution is to specify a path (either via an absolute path or some sort
760 of path insertion or path appending mechanism) that would apply to the
761 C{executeCommand()} function. This is not difficult to implement, but it
762 seem like kind of a "big hammer" solution. Besides that, it might also
763 represent a security flaw (for instance, I prefer not to mess with root's
764 C{$PATH} on the application level if I don't have to).
765
766 The alternative is to set up some sort of configuration for the path to
767 certain executables, i.e. "find C{svnlook} in C{/usr/local/bin/svnlook}" or
768 whatever. This PathResolverSingleton aims to provide a good solution to the
769 mapping problem. Callers of all sorts (extensions or not) can get an
770 instance of the singleton. Then, they call the C{lookup} method to try and
771 resolve the executable they are looking for. Through the C{lookup} method,
772 the caller can also specify a default to use if a mapping is not found.
773 This way, with no real effort on the part of the caller, behavior can neatly
774 degrade to something equivalent to the current behavior if there is no
775 special mapping or if the singleton was never initialized in the first
776 place.
777
778 Even better, extensions automagically get access to the same resolver
779 functionality, and they don't even need to understand how the mapping
780 happens. All extension authors need to do is document what executables
781 their code requires, and the standard resolver configuration section will
782 meet their needs.
783
784 The class should be initialized once through the constructor somewhere in
785 the main routine. Then, the main routine should call the L{fill} method to
786 fill in the resolver's internal structures. Everyone else who needs to
787 resolve a path will get an instance of the class using L{getInstance} and
788 will then just call the L{lookup} method.
789
790 @cvar _instance: Holds a reference to the singleton
791 @ivar _mapping: Internal mapping from resource name to path.
792 """
793
794 _instance = None
795
797 """Helper class to provide a singleton factory method."""
803
804 getInstance = _Helper()
805
812
813 - def lookup(self, name, default=None):
814 """
815 Looks up name and returns the resolved path associated with the name.
816 @param name: Name of the path resource to resolve.
817 @param default: Default to return if resource cannot be resolved.
818 @return: Resolved path associated with name, or default if name can't be resolved.
819 """
820 value = default
821 if name in self._mapping.keys():
822 value = self._mapping[name]
823 logger.debug("Resolved command [%s] to [%s]." % (name, value))
824 return value
825
826 - def fill(self, mapping):
827 """
828 Fills in the singleton's internal mapping from name to resource.
829 @param mapping: Mapping from resource name to path.
830 @type mapping: Dictionary mapping name to path, both as strings.
831 """
832 self._mapping = { }
833 for key in mapping.keys():
834 self._mapping[key] = mapping[key]
835
836
837
838
839
840
841 if _PIPE_IMPLEMENTATION == "subprocess.Popen":
842
843 from subprocess import STDOUT, PIPE
844
846 """
847 Specialized pipe class for use by C{executeCommand}.
848
849 The L{executeCommand} function needs a specialized way of interacting
850 with a pipe. First, C{executeCommand} only reads from the pipe, and
851 never writes to it. Second, C{executeCommand} needs a way to discard all
852 output written to C{stderr}, as a means of simulating the shell
853 C{2>/dev/null} construct.
854
855 All of this functionality is provided (in Python 2.4 or later) by the
856 C{subprocess.Popen} class, so when that class is available, we'll use it.
857 Otherwise, there's another implementation based on C{popen2.Popen4},
858 which unfortunately only works on UNIX platforms.
859 """
860 - def __init__(self, cmd, bufsize=-1, ignoreStderr=False):
861 stderr = STDOUT
862 if ignoreStderr:
863 devnull = nullDevice()
864 stderr = os.open(devnull, os.O_RDWR)
865 Popen.__init__(self, shell=False, args=cmd, bufsize=bufsize, stdin=None, stdout=PIPE, stderr=stderr)
866 self.fromchild = self.stdout
867
868 else:
869
870 from popen2 import _cleanup, _active
871
872 - class Pipe(Popen4):
873 """
874 Specialized pipe class for use by C{executeCommand}.
875
876 The L{executeCommand} function needs a specialized way of interacting with a
877 pipe that isn't satisfied by the standard C{Popen3} and C{Popen4} classes in
878 C{popen2}. First, C{executeCommand} only reads from the pipe, and never
879 writes to it. Second, C{executeCommand} needs a way to discard all output
880 written to C{stderr}, as a means of simulating the shell C{2>/dev/null}
881 construct.
882
883 This class inherits from C{Popen4}. If the C{ignoreStderr} flag is passed in
884 as C{False}, then the standard C{Popen4} constructor will be called and
885 C{stdout} and C{stderr} will be intermingled in the output.
886
887 Otherwise, we'll call a custom version of the constructor which was
888 basically stolen from the real constructor in C{python2.3/Lib/popen2.py}.
889 This custom constructor will redirect the C{stderr} file descriptor to
890 C{/dev/null}. I've done this based on a suggestion from Donn Cave on
891 comp.lang.python.
892
893 In either case, the C{tochild} file object is always closed before returning
894 from the constructor, since it is never needed by C{executeCommand}.
895
896 I really wish there were a prettier way to do this. Unfortunately, I
897 need access to the guts of the constructor implementation because of the
898 way the pipe process is forked, etc. It doesn't work to just call the
899 superclass constructor and then modify a few things afterwards. Even
900 worse, I have to access private C{popen2} module members C{_cleanup} and
901 C{_active} in order to duplicate the implementation.
902
903 Hopefully this whole thing will continue to work properly. At least we
904 can use the other L{subprocess.Popen}-based implementation when that
905 class is available.
906
907 @copyright: Some of this code, prior to customization, was originally part
908 of the Python 2.3 codebase. Python code is copyright (c) 2001, 2002 Python
909 Software Foundation; All Rights Reserved.
910 """
911
912 - def __init__(self, cmd, bufsize=-1, ignoreStderr=False):
913 if not ignoreStderr:
914 Popen4.__init__(self, cmd, bufsize)
915 else:
916 _cleanup()
917 p2cread, p2cwrite = os.pipe()
918 c2pread, c2pwrite = os.pipe()
919 self.pid = os.fork()
920 if self.pid == 0:
921 os.dup2(p2cread, 0)
922 os.dup2(c2pwrite, 1)
923 devnull = nullDevice()
924 null = os.open(devnull, os.O_RDWR)
925 os.dup2(null, 2)
926 os.close(null)
927 self._run_child(cmd)
928 os.close(p2cread)
929 self.tochild = os.fdopen(p2cwrite, 'w', bufsize)
930 os.close(c2pwrite)
931 self.fromchild = os.fdopen(c2pread, 'r', bufsize)
932 _active.append(self)
933 self.tochild.close()
934
935
936
937
938
939
940
941
942
943
945 """
946 Returns the keys of the dictionary sorted by value.
947 There are cuter ways to do this in Python 2.4, but we're compatible with 2.3.
948 @param d: Dictionary to operate on
949 @return: List of dictionary keys sorted in order by dictionary value.
950 """
951 items = d.items()
952 items.sort(lambda x, y: cmp(x[1], y[1]))
953 return [key for key, value in items]
954
955
956
957
958
959
961 """
962 Removes all of the keys from the dictionary.
963 The dictionary is altered in-place.
964 Each key must exist in the dictionary.
965 @param d: Dictionary to operate on
966 @param keys: List of keys to remove
967 @raise KeyError: If one of the keys does not exist
968 """
969 for key in keys:
970 del d[key]
971
972
973
974
975
976
978 """
979 Converts a size in one unit to a size in another unit.
980
981 This is just a convenience function so that the functionality can be
982 implemented in just one place. Internally, we convert values to bytes and
983 then to the final unit.
984
985 The available units are:
986
987 - C{UNIT_BYTES} - Bytes
988 - C{UNIT_KBYTES} - Kilobytes, where 1 kB = 1024 B
989 - C{UNIT_MBYTES} - Megabytes, where 1 MB = 1024 kB
990 - C{UNIT_GBYTES} - Gigabytes, where 1 GB = 1024 MB
991 - C{UNIT_SECTORS} - Sectors, where 1 sector = 2048 B
992
993 @param size: Size to convert
994 @type size: Integer or float value in units of C{fromUnit}
995
996 @param fromUnit: Unit to convert from
997 @type fromUnit: One of the units listed above
998
999 @param toUnit: Unit to convert to
1000 @type toUnit: One of the units listed above
1001
1002 @return: Number converted to new unit, as a float.
1003 @raise ValueError: If one of the units is invalid.
1004 """
1005 if size is None:
1006 raise ValueError("Cannot convert size of None.")
1007 if fromUnit == UNIT_BYTES:
1008 byteSize = float(size)
1009 elif fromUnit == UNIT_KBYTES:
1010 byteSize = float(size) * BYTES_PER_KBYTE
1011 elif fromUnit == UNIT_MBYTES:
1012 byteSize = float(size) * BYTES_PER_MBYTE
1013 elif fromUnit == UNIT_GBYTES:
1014 byteSize = float(size) * BYTES_PER_GBYTE
1015 elif fromUnit == UNIT_SECTORS:
1016 byteSize = float(size) * BYTES_PER_SECTOR
1017 else:
1018 raise ValueError("Unknown 'from' unit %s." % fromUnit)
1019 if toUnit == UNIT_BYTES:
1020 return byteSize
1021 elif toUnit == UNIT_KBYTES:
1022 return byteSize / BYTES_PER_KBYTE
1023 elif toUnit == UNIT_MBYTES:
1024 return byteSize / BYTES_PER_MBYTE
1025 elif toUnit == UNIT_GBYTES:
1026 return byteSize / BYTES_PER_GBYTE
1027 elif toUnit == UNIT_SECTORS:
1028 return byteSize / BYTES_PER_SECTOR
1029 else:
1030 raise ValueError("Unknown 'to' unit %s." % toUnit)
1031
1032
1033
1034
1035
1036
1038 """
1039 Format a byte quantity so it can be sensibly displayed.
1040
1041 It's rather difficult to look at a number like "72372224 bytes" and get any
1042 meaningful information out of it. It would be more useful to see something
1043 like "69.02 MB". That's what this function does. Any time you want to display
1044 a byte value, i.e.::
1045
1046 print "Size: %s bytes" % bytes
1047
1048 Call this function instead::
1049
1050 print "Size: %s" % displayBytes(bytes)
1051
1052 What comes out will be sensibly formatted. The indicated number of digits
1053 will be listed after the decimal point, rounded based on whatever rules are
1054 used by Python's standard C{%f} string format specifier. (Values less than 1
1055 kB will be listed in bytes and will not have a decimal point, since the
1056 concept of a fractional byte is nonsensical.)
1057
1058 @param bytes: Byte quantity.
1059 @type bytes: Integer number of bytes.
1060
1061 @param digits: Number of digits to display after the decimal point.
1062 @type digits: Integer value, typically 2-5.
1063
1064 @return: String, formatted for sensible display.
1065 """
1066 if(bytes is None):
1067 raise ValueError("Cannot display byte value of None.")
1068 bytes = float(bytes)
1069 if math.fabs(bytes) < BYTES_PER_KBYTE:
1070 format = "%.0f bytes"
1071 value = bytes
1072 elif math.fabs(bytes) < BYTES_PER_MBYTE:
1073 format = "%." + "%d" % digits + "f kB"
1074 value = bytes / BYTES_PER_KBYTE
1075 elif math.fabs(bytes) < BYTES_PER_GBYTE:
1076 format = "%." + "%d" % digits + "f MB"
1077 value = bytes / BYTES_PER_MBYTE
1078 else:
1079 format = "%." + "%d" % digits + "f GB"
1080 value = bytes / BYTES_PER_GBYTE
1081 return format % value
1082
1083
1084
1085
1086
1087
1089 """
1090 Gets a reference to a named function.
1091
1092 This does some hokey-pokey to get back a reference to a dynamically named
1093 function. For instance, say you wanted to get a reference to the
1094 C{os.path.isdir} function. You could use::
1095
1096 myfunc = getFunctionReference("os.path", "isdir")
1097
1098 Although we won't bomb out directly, behavior is pretty much undefined if
1099 you pass in C{None} or C{""} for either C{module} or C{function}.
1100
1101 The only validation we enforce is that whatever we get back must be
1102 callable.
1103
1104 I derived this code based on the internals of the Python unittest
1105 implementation. I don't claim to completely understand how it works.
1106
1107 @param module: Name of module associated with function.
1108 @type module: Something like "os.path" or "CedarBackup2.util"
1109
1110 @param function: Name of function
1111 @type function: Something like "isdir" or "getUidGid"
1112
1113 @return: Reference to function associated with name.
1114
1115 @raise ImportError: If the function cannot be found.
1116 @raise ValueError: If the resulting reference is not callable.
1117
1118 @copyright: Some of this code, prior to customization, was originally part
1119 of the Python 2.3 codebase. Python code is copyright (c) 2001, 2002 Python
1120 Software Foundation; All Rights Reserved.
1121 """
1122 parts = []
1123 if module is not None and module != "":
1124 parts = module.split(".")
1125 if function is not None and function != "":
1126 parts.append(function);
1127 copy = parts[:]
1128 while copy:
1129 try:
1130 module = __import__(string.join(copy, "."))
1131 break
1132 except ImportError:
1133 del copy[-1]
1134 if not copy: raise
1135 parts = parts[1:]
1136 obj = module
1137 for part in parts:
1138 obj = getattr(obj, part)
1139 if not callable(obj):
1140 raise ValueError("Reference to %s.%s is not callable." % (module, function))
1141 return obj
1142
1143
1144
1145
1146
1147
1149 """
1150 Get the uid/gid associated with a user/group pair
1151
1152 This is a no-op if user/group functionality is not available on the platform.
1153
1154 @param user: User name
1155 @type user: User name as a string
1156
1157 @param group: Group name
1158 @type group: Group name as a string
1159
1160 @return: Tuple C{(uid, gid)} matching passed-in user and group.
1161 @raise ValueError: If the ownership user/group values are invalid
1162 """
1163 if _UID_GID_AVAILABLE:
1164 try:
1165 uid = pwd.getpwnam(user)[2]
1166 gid = grp.getgrnam(group)[2]
1167 logger.debug("Translated [%s:%s] into [%d:%d]." % (user, group, uid, gid))
1168 return (uid, gid)
1169 except Exception, e:
1170 logger.debug("Error looking up uid and gid for [%s:%s]: %s" % (user, group, e))
1171 raise ValueError("Unable to lookup up uid and gid for passed in user/group.")
1172 else:
1173 return (0,0)
1174
1175
1176
1177
1178
1179
1181 """
1182 Changes ownership of path to match the user and group.
1183
1184 This is a no-op if user/group functionality is not available on the
1185 platform, or if the either passed-in user or group is C{None}.
1186
1187 @param path: Path whose ownership to change.
1188 @param user: User which owns file.
1189 @param group: Group which owns file.
1190 """
1191 if _UID_GID_AVAILABLE:
1192 if user is None or group is None:
1193 logger.debug("User or group is None, so not attempting to change owner on [%s]." % path)
1194 elif os.getuid() != 0:
1195 logger.debug("Not root, so not attempting to change owner on [%s]." % path)
1196 else:
1197 try:
1198 (uid, gid) = getUidGid(user, group)
1199 os.chown(path, uid, gid)
1200 except Exception, e:
1201 logger.error("Error changing ownership of [%s]: %s" % (path, e))
1202
1203
1204
1205
1206
1207
1209 """
1210 Splits a command line string into a list of arguments.
1211
1212 Unfortunately, there is no "standard" way to parse a command line string,
1213 and it's actually not an easy problem to solve portably (essentially, we
1214 have to emulate the shell argument-processing logic). This code only
1215 respects double quotes (C{"}) for grouping arguments, not single quotes
1216 (C{'}). Make sure you take this into account when building your command
1217 line.
1218
1219 Incidentally, I found this particular parsing method while digging around in
1220 Google Groups, and I tweaked it for my own use.
1221
1222 @param commandLine: Command line string
1223 @type commandLine: String, i.e. "cback --verbose stage store"
1224
1225 @return: List of arguments, suitable for passing to C{popen2}.
1226
1227 @raise ValueError: If the command line is None.
1228 """
1229 if commandLine is None:
1230 raise ValueError("Cannot split command line of None.")
1231 fields = re.findall('[^ "]+|"[^"]+"', commandLine)
1232 fields = map(lambda field: field.replace('"', ''), fields)
1233 return fields
1234
1235
1236
1237
1238
1239
1241 """
1242 Resolves the real path to a command through the path resolver mechanism.
1243
1244 Both extensions and standard Cedar Backup functionality need a way to
1245 resolve the "real" location of various executables. Normally, they assume
1246 that these executables are on the system path, but some callers need to
1247 specify an alternate location.
1248
1249 Ideally, we want to handle this configuration in a central location. The
1250 Cedar Backup path resolver mechanism (a singleton called
1251 L{PathResolverSingleton}) provides the central location to store the
1252 mappings. This function wraps access to the singleton, and is what all
1253 functions (extensions or standard functionality) should call if they need to
1254 find a command.
1255
1256 The passed-in command must actually be a list, in the standard form used by
1257 all existing Cedar Backup code (something like C{["svnlook", ]}). The
1258 lookup will actually be done on the first element in the list, and the
1259 returned command will always be in list form as well.
1260
1261 If the passed-in command can't be resolved or no mapping exists, then the
1262 command itself will be returned unchanged. This way, we neatly fall back on
1263 default behavior if we have no sensible alternative.
1264
1265 @param command: Command to resolve.
1266 @type command: List form of command, i.e. C{["svnlook", ]}.
1267
1268 @return: Path to command or just command itself if no mapping exists.
1269 """
1270 singleton = PathResolverSingleton.getInstance()
1271 name = command[0]
1272 result = command[:]
1273 result[0] = singleton.lookup(name, name)
1274 return result
1275
1276
1277
1278
1279
1280
1281 -def executeCommand(command, args, returnOutput=False, ignoreStderr=False, doNotLog=False, outputFile=None):
1282 """
1283 Executes a shell command, hopefully in a safe way.
1284
1285 This function exists to replace direct calls to C{os.popen} in the Cedar
1286 Backup code. It's not safe to call a function such as C{os.popen()} with
1287 untrusted arguments, since that can cause problems if the string contains
1288 non-safe variables or other constructs (imagine that the argument is
1289 C{$WHATEVER}, but C{$WHATEVER} contains something like C{"; rm -fR ~/;
1290 echo"} in the current environment).
1291
1292 Instead, it's safer to pass a list of arguments in the style supported bt
1293 C{popen2} or C{popen4}. This function actually uses a specialized C{Pipe}
1294 class implemented using either C{subprocess.Popen} or C{popen2.Popen4}.
1295
1296 Under the normal case, this function will return a tuple of C{(status,
1297 None)} where the status is the wait-encoded return status of the call per
1298 the C{popen2.Popen4} documentation. If C{returnOutput} is passed in as
1299 C{True}, the function will return a tuple of C{(status, output)} where
1300 C{output} is a list of strings, one entry per line in the output from the
1301 command. Output is always logged to the C{outputLogger.info()} target,
1302 regardless of whether it's returned.
1303
1304 By default, C{stdout} and C{stderr} will be intermingled in the output.
1305 However, if you pass in C{ignoreStderr=True}, then only C{stdout} will be
1306 included in the output.
1307
1308 The C{doNotLog} parameter exists so that callers can force the function to
1309 not log command output to the debug log. Normally, you would want to log.
1310 However, if you're using this function to write huge output files (i.e.
1311 database backups written to C{stdout}) then you might want to avoid putting
1312 all that information into the debug log.
1313
1314 The C{outputFile} parameter exists to make it easier for a caller to push
1315 output into a file, i.e. as a substitute for redirection to a file. If this
1316 value is passed in, each time a line of output is generated, it will be
1317 written to the file using C{outputFile.write()}. At the end, the file
1318 descriptor will be flushed using C{outputFile.flush()}. The caller
1319 maintains responsibility for closing the file object appropriately.
1320
1321 @note: I know that it's a bit confusing that the command and the arguments
1322 are both lists. I could have just required the caller to pass in one big
1323 list. However, I think it makes some sense to keep the command (the
1324 constant part of what we're executing, i.e. C{"scp -B"}) separate from its
1325 arguments, even if they both end up looking kind of similar.
1326
1327 @note: You cannot redirect output via shell constructs (i.e. C{>file},
1328 C{2>/dev/null}, etc.) using this function. The redirection string would be
1329 passed to the command just like any other argument. However, you can
1330 implement the equivalent to redirection using C{ignoreStderr} and
1331 C{outputFile}, as discussed above.
1332
1333 @param command: Shell command to execute
1334 @type command: List of individual arguments that make up the command
1335
1336 @param args: List of arguments to the command
1337 @type args: List of additional arguments to the command
1338
1339 @param returnOutput: Indicates whether to return the output of the command
1340 @type returnOutput: Boolean C{True} or C{False}
1341
1342 @param doNotLog: Indicates that output should not be logged.
1343 @type doNotLog: Boolean C{True} or C{False}
1344
1345 @param outputFile: File object that all output should be written to.
1346 @type outputFile: File object as returned from C{open()} or C{file()}.
1347
1348 @return: Tuple of C{(result, output)} as described above.
1349 """
1350 logger.debug("Executing command %s with args %s." % (command, args))
1351 outputLogger.info("Executing command %s with args %s." % (command, args))
1352 if doNotLog:
1353 logger.debug("Note: output will not be logged, per the doNotLog flag.")
1354 outputLogger.info("Note: output will not be logged, per the doNotLog flag.")
1355 output = []
1356 fields = command[:]
1357 fields.extend(args)
1358 try:
1359 pipe = Pipe(fields, ignoreStderr=ignoreStderr)
1360 while True:
1361 line = pipe.fromchild.readline()
1362 if not line: break
1363 if returnOutput: output.append(line)
1364 if outputFile is not None: outputFile.write(line)
1365 if not doNotLog: outputLogger.info(line[:-1])
1366 if outputFile is not None:
1367 try:
1368 outputFile.flush()
1369 except: pass
1370 if returnOutput:
1371 return (pipe.wait(), output)
1372 else:
1373 return (pipe.wait(), None)
1374 except OSError, e:
1375 try:
1376 if returnOutput:
1377 if output != []:
1378 return (pipe.wait(), output)
1379 else:
1380 return (pipe.wait(), [ e, ])
1381 else:
1382 return (pipe.wait(), None)
1383 except UnboundLocalError:
1384 if returnOutput:
1385 return (256, [])
1386 else:
1387 return (256, None)
1388
1389
1390
1391
1392
1393
1395 """
1396 Calculates the age (in days) of a file.
1397
1398 The "age" of a file is the amount of time since the file was last used, per
1399 the most recent of the file's C{st_atime} and C{st_mtime} values.
1400
1401 Technically, we only intend this function to work with files, but it will
1402 probably work with anything on the filesystem.
1403
1404 @param file: Path to a file on disk.
1405
1406 @return: Age of the file in days.
1407 @raise OSError: If the file doesn't exist.
1408 """
1409 currentTime = int(time.time())
1410 fileStats = os.stat(file)
1411 lastUse = max(fileStats.st_atime, fileStats.st_mtime)
1412 ageInDays = (currentTime - lastUse) / SECONDS_PER_DAY
1413 return ageInDays
1414
1415
1416
1417
1418
1419
1420 -def mount(devicePath, mountPoint, fsType):
1421 """
1422 Mounts the indicated device at the indicated mount point.
1423
1424 For instance, to mount a CD, you might use device path C{/dev/cdrw}, mount
1425 point C{/media/cdrw} and filesystem type C{iso9660}. You can safely use any
1426 filesystem type that is supported by C{mount} on your platform. If the type
1427 is C{None}, we'll attempt to let C{mount} auto-detect it. This may or may
1428 not work on all systems.
1429
1430 @note: This only works on platforms that have a concept of "mounting" a
1431 filesystem through a command-line C{"mount"} command, like UNIXes. It
1432 won't work on Windows.
1433
1434 @param devicePath: Path of device to be mounted.
1435 @param mountPoint: Path that device should be mounted at.
1436 @param fsType: Type of the filesystem assumed to be available via the device.
1437
1438 @raise IOError: If the device cannot be mounted.
1439 """
1440 if fsType is None:
1441 args = [ devicePath, mountPoint ]
1442 else:
1443 args = [ "-t", fsType, devicePath, mountPoint ]
1444 command = resolveCommand(MOUNT_COMMAND)
1445 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True)[0]
1446 if result != 0:
1447 raise IOError("Error [%d] mounting [%s] at [%s] as [%s]." % (result, devicePath, mountPoint, fsType))
1448
1449
1450
1451
1452
1453
1454 -def unmount(mountPoint, removeAfter=False, attempts=1, waitSeconds=0):
1455 """
1456 Unmounts whatever device is mounted at the indicated mount point.
1457
1458 Sometimes, it might not be possible to unmount the mount point immediately,
1459 if there are still files open there. Use the C{attempts} and C{waitSeconds}
1460 arguments to indicate how many unmount attempts to make and how many seconds
1461 to wait between attempts. If you pass in zero attempts, no attempts will be
1462 made (duh).
1463
1464 If the indicated mount point is not really a mount point per
1465 C{os.path.ismount()}, then it will be ignored. This seems to be a safer
1466 check then looking through C{/etc/mtab}, since C{ismount()} is already in
1467 the Python standard library and is documented as working on all POSIX
1468 systems.
1469
1470 If C{removeAfter} is C{True}, then the mount point will be removed using
1471 C{os.rmdir()} after the unmount action succeeds. If for some reason the
1472 mount point is not a directory, then it will not be removed.
1473
1474 @note: This only works on platforms that have a concept of "mounting" a
1475 filesystem through a command-line C{"mount"} command, like UNIXes. It
1476 won't work on Windows.
1477
1478 @param mountPoint: Mount point to be unmounted.
1479 @param removeAfter: Remove the mount point after unmounting it.
1480 @param attempts: Number of times to attempt the unmount.
1481 @param waitSeconds: Number of seconds to wait between repeated attempts.
1482
1483 @raise IOError: If the mount point is still mounted after attempts are exhausted.
1484 """
1485 if os.path.ismount(mountPoint):
1486 for attempt in range(0, attempts):
1487 logger.debug("Making attempt %d to unmount [%s]." % (attempt, mountPoint))
1488 command = resolveCommand(UMOUNT_COMMAND)
1489 result = executeCommand(command, [ mountPoint, ], returnOutput=False, ignoreStderr=True)[0]
1490 if result != 0:
1491 logger.error("Error [%d] unmounting [%s] on attempt %d." % (result, mountPoint, attempt))
1492 elif os.path.ismount(mountPoint):
1493 logger.error("After attempt %d, [%s] is still mounted." % (attempt, mountPoint))
1494 else:
1495 logger.debug("Successfully unmounted [%s] on attempt %d." % (mountPoint, attempt))
1496 break
1497 if attempt+1 < attempts:
1498 if waitSeconds > 0:
1499 logger.info("Sleeping %d second(s) before next unmount attempt." % waitSeconds)
1500 time.sleep(waitSeconds)
1501 else:
1502 if os.path.ismount(mountPoint):
1503 raise IOError("Unable to unmount [%s] after %d attempts." % (mountPoint, attempts))
1504 logger.info("Mount point [%s] seems to have finally gone away." % mountPoint)
1505 if os.path.isdir(mountPoint) and removeAfter:
1506 logger.debug("Removing mount point [%s]." % mountPoint)
1507 os.rmdir(mountPoint)
1508
1509
1510
1511
1512
1513
1515 """
1516 Indicates whether a specific filesystem device is currently mounted.
1517
1518 We determine whether the device is mounted by looking through the system's
1519 C{mtab} file. This file shows every currently-mounted filesystem, ordered
1520 by device. We only do the check if the C{mtab} file exists and is readable.
1521 Otherwise, we assume that the device is not mounted.
1522
1523 @note: This only works on platforms that have a concept of an mtab file
1524 to show mounted volumes, like UNIXes. It won't work on Windows.
1525
1526 @param devicePath: Path of device to be checked
1527
1528 @return: True if device is mounted, false otherwise.
1529 """
1530 if os.path.exists(MTAB_FILE) and os.access(MTAB_FILE, os.R_OK):
1531 realPath = os.path.realpath(devicePath)
1532 lines = open(MTAB_FILE).readlines()
1533 for line in lines:
1534 (mountDevice, mountPoint, remainder) = line.split(None, 2)
1535 if mountDevice in [ devicePath, realPath, ]:
1536 logger.debug("Device [%s] is mounted at [%s]." % (devicePath, mountPoint))
1537 return True
1538 return False
1539
1540
1541
1542
1543
1544
1546
1547 """
1548 Safely encodes a filesystem path.
1549
1550 Many Python filesystem functions, such as C{os.listdir}, behave differently
1551 if they are passed unicode arguments versus simple string arguments. For
1552 instance, C{os.listdir} generally returns unicode path names if it is passed
1553 a unicode argument, and string pathnames if it is passed a string argument.
1554
1555 However, this behavior often isn't as consistent as we might like. As an example,
1556 C{os.listdir} "gives up" if it finds a filename that it can't properly encode
1557 given the current locale settings. This means that the returned list is
1558 a mixed set of unicode and simple string paths. This has consequences later,
1559 because other filesystem functions like C{os.path.join} will blow up if they
1560 are given one string path and one unicode path.
1561
1562 On comp.lang.python, Martin v. Löwis explained the C{os.listdir} behavior
1563 like this::
1564
1565 The operating system (POSIX) does not have the inherent notion that file
1566 names are character strings. Instead, in POSIX, file names are primarily
1567 byte strings. There are some bytes which are interpreted as characters
1568 (e.g. '\x2e', which is '.', or '\x2f', which is '/'), but apart from
1569 that, most OS layers think these are just bytes.
1570
1571 Now, most *people* think that file names are character strings. To
1572 interpret a file name as a character string, you need to know what the
1573 encoding is to interpret the file names (which are byte strings) as
1574 character strings.
1575
1576 There is, unfortunately, no operating system API to carry the notion of a
1577 file system encoding. By convention, the locale settings should be used
1578 to establish this encoding, in particular the LC_CTYPE facet of the
1579 locale. This is defined in the environment variables LC_CTYPE, LC_ALL,
1580 and LANG (searched in this order).
1581
1582 If LANG is not set, the "C" locale is assumed, which uses ASCII as its
1583 file system encoding. In this locale, '\xe2\x99\xaa\xe2\x99\xac' is not a
1584 valid file name (at least it cannot be interpreted as characters, and
1585 hence not be converted to Unicode).
1586
1587 Now, your Python script has requested that all file names *should* be
1588 returned as character (ie. Unicode) strings, but Python cannot comply,
1589 since there is no way to find out what this byte string means, in terms
1590 of characters.
1591
1592 So we have three options:
1593
1594 1. Skip this string, only return the ones that can be converted to Unicode.
1595 Give the user the impression the file does not exist.
1596 2. Return the string as a byte string
1597 3. Refuse to listdir altogether, raising an exception (i.e. return nothing)
1598
1599 Python has chosen alternative 2, allowing the application to implement 1
1600 or 3 on top of that if it wants to (or come up with other strategies,
1601 such as user feedback).
1602
1603 As a solution, he suggests that rather than passing unicode paths into the
1604 filesystem functions, that I should sensibly encode the path first. That is
1605 what this function accomplishes. Any function which takes a filesystem path
1606 as an argument should encode it first, before using it for any other purpose.
1607
1608 I confess I still don't completely understand how this works. On a system
1609 with filesystem encoding "ISO-8859-1", a path C{u"\xe2\x99\xaa\xe2\x99\xac"}
1610 is converted into the string C{"\xe2\x99\xaa\xe2\x99\xac"}. However, on a
1611 system with a "utf-8" encoding, the result is a completely different string:
1612 C{"\xc3\xa2\xc2\x99\xc2\xaa\xc3\xa2\xc2\x99\xc2\xac"}. A quick test where I
1613 write to the first filename and open the second proves that the two strings
1614 represent the same file on disk, which is all I really care about.
1615
1616 @note: As a special case, if C{path} is C{None}, then this function will
1617 return C{None}.
1618
1619 @note: To provide several examples of encoding values, my Debian sarge box
1620 with an ext3 filesystem has Python filesystem encoding C{ISO-8859-1}. User
1621 Anarcat's Debian box with a xfs filesystem has filesystem encoding
1622 C{ANSI_X3.4-1968}. Both my iBook G4 running Mac OS X 10.4 and user Dag
1623 Rende's SuSE 9.3 box both have filesystem encoding C{UTF-8}.
1624
1625 @note: Just because a filesystem has C{UTF-8} encoding doesn't mean that it
1626 will be able to handle all extended-character filenames. For instance,
1627 certain extended-character (but not UTF-8) filenames -- like the ones in the
1628 regression test tar file C{test/data/tree13.tar.gz} -- are not valid under
1629 Mac OS X, and it's not even possible to extract them from the tarfile on
1630 that platform.
1631
1632 @param path: Path to encode
1633
1634 @return: Path, as a string, encoded appropriately
1635 @raise ValueError: If the path cannot be encoded properly.
1636 """
1637 if path is None:
1638 return path
1639 try:
1640 if isinstance(path, unicode):
1641 encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
1642 path = path.encode(encoding)
1643 return path
1644 except UnicodeError:
1645 raise ValueError("Path could not be safely encoded as %s." % encoding)
1646
1647
1648
1649
1650
1651
1653 """
1654 Attempts to portably return the null device on this system.
1655
1656 The null device is something like C{/dev/null} on a UNIX system. The name
1657 varies on other platforms.
1658
1659 In Python 2.4 and better, we can use C{os.devnull}. Since we want to be
1660 portable to python 2.3, getting the value in earlier versions of Python
1661 takes some screwing around. Basically, this function will only work on
1662 either UNIX-like systems (the default) or Windows.
1663 """
1664 try:
1665 return os.devnull
1666 except AttributeError:
1667 import platform
1668 if platform.platform().startswith("Windows"):
1669 return "NUL"
1670 else:
1671 return "/dev/null"
1672
1673
1674
1675
1676
1677
1679 """
1680 Converts English day name to numeric day of week as from C{time.localtime}.
1681
1682 For instance, the day C{monday} would be converted to the number C{0}.
1683
1684 @param dayName: Day of week to convert
1685 @type dayName: string, i.e. C{"monday"}, C{"tuesday"}, etc.
1686
1687 @returns: Integer, where Monday is 0 and Sunday is 6; or -1 if no conversion is possible.
1688 """
1689 if dayName.lower() == "monday":
1690 return 0
1691 elif dayName.lower() == "tuesday":
1692 return 1
1693 elif dayName.lower() == "wednesday":
1694 return 2
1695 elif dayName.lower() == "thursday":
1696 return 3
1697 elif dayName.lower() == "friday":
1698 return 4
1699 elif dayName.lower() == "saturday":
1700 return 5
1701 elif dayName.lower() == "sunday":
1702 return 6
1703 else:
1704 return -1
1705
1706
1707
1708
1709
1710
1712 """
1713 Indicates whether "today" is the backup starting day per configuration.
1714
1715 If the current day's English name matches the indicated starting day, then
1716 today is a starting day.
1717
1718 @param startingDay: Configured starting day.
1719 @type startingDay: string, i.e. C{"monday"}, C{"tuesday"}, etc.
1720
1721 @return: Boolean indicating whether today is the starting day.
1722 """
1723 value = time.localtime().tm_wday == deriveDayOfWeek(startingDay)
1724 if value:
1725 logger.debug("Today is the start of the week.")
1726 else:
1727 logger.debug("Today is NOT the start of the week.")
1728 return value
1729
1730
1731
1732
1733
1734
1736 """
1737 Returns a "normalized" path based on a path name.
1738
1739 A normalized path is a representation of a path that is also a valid file
1740 name. To make a valid file name out of a complete path, we have to convert
1741 or remove some characters that are significant to the filesystem -- in
1742 particular, the path separator and any leading C{'.'} character (which would
1743 cause the file to be hidden in a file listing).
1744
1745 Note that this is a one-way transformation -- you can't safely derive the
1746 original path from the normalized path.
1747
1748 To normalize a path, we begin by looking at the first character. If the
1749 first character is C{'/'} or C{'\\'}, it gets removed. If the first
1750 character is C{'.'}, it gets converted to C{'_'}. Then, we look through the
1751 rest of the path and convert all remaining C{'/'} or C{'\\'} characters
1752 C{'-'}, and all remaining whitespace characters to C{'_'}.
1753
1754 As a special case, a path consisting only of a single C{'/'} or C{'\\'}
1755 character will be converted to C{'-'}.
1756
1757 @param path: Path to normalize
1758
1759 @return: Normalized path as described above.
1760
1761 @raise ValueError: If the path is None
1762 """
1763 if path is None:
1764 raise ValueError("Cannot normalize path None.")
1765 elif len(path) == 0:
1766 return path
1767 elif path == "/" or path == "\\":
1768 return "-"
1769 else:
1770 normalized = path
1771 normalized = re.sub(r"^\/", "", normalized)
1772 normalized = re.sub(r"^\\", "", normalized)
1773 normalized = re.sub(r"^\.", "_", normalized)
1774 normalized = re.sub(r"\/", "-", normalized)
1775 normalized = re.sub(r"\\", "-", normalized)
1776 normalized = re.sub(r"\s", "_", normalized)
1777 return normalized
1778