CedarBackup2.peer

89 90 ###################### 91 # Class documentation 92 ###################### 93 94 """ 95 Backup peer representing a local peer in a backup pool. 96 97 This is a class representing a local (non-network) peer in a backup pool. 98 Local peers are backed up by simple filesystem copy operations. A local 99 peer has associated with it a name (typically, but not necessarily, a 100 hostname) and a collect directory. 101 102 The public methods other than the constructor are part of a "backup peer" 103 interface shared with the C{RemotePeer} class. 104 105 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator, 106 _copyLocalDir, _copyLocalFile, name, collectDir 107 """ 108 109 ############## 110 # Constructor 111 ############## 112

113 - def __init__(self, name, collectDir):

114 """ 115 Initializes a local backup peer. 116 117 Note that the collect directory must be an absolute path, but does not 118 have to exist when the object is instantiated. We do a lazy validation 119 on this value since we could (potentially) be creating peer objects 120 before an ongoing backup completed. 121 122 @param name: Name of the backup peer 123 @type name: String, typically a hostname 124 125 @param collectDir: Path to the peer's collect directory 126 @type collectDir: String representing an absolute local path on disk 127 128 @raise ValueError: If the name is empty. 129 @raise ValueError: If collect directory is not an absolute path. 130 """ 131 self._name = None 132 self._collectDir = None 133 self.name = name 134 self.collectDir = collectDir

135 136 137 ############# 138 # Properties 139 ############# 140

141 - def _setName(self, value):

142 """ 143 Property target used to set the peer name. 144 The value must be a non-empty string and cannot be C{None}. 145 @raise ValueError: If the value is an empty string or C{None}. 146 """ 147 if value is None or len(value) < 1: 148 raise ValueError("Peer name must be a non-empty string.") 149 self._name = value

150

151 - def _getName(self):

152 """ 153 Property target used to get the peer name. 154 """ 155 return self._name

156

157 - def _setCollectDir(self, value):

158 """ 159 Property target used to set the collect directory. 160 The value must be an absolute path and cannot be C{None}. 161 It does not have to exist on disk at the time of assignment. 162 @raise ValueError: If the value is C{None} or is not an absolute path. 163 @raise ValueError: If a path cannot be encoded properly. 164 """ 165 if value is None or not os.path.isabs(value): 166 raise ValueError("Collect directory must be an absolute path.") 167 self._collectDir = encodePath(value)

168

169 - def _getCollectDir(self):

170 """ 171 Property target used to get the collect directory. 172 """ 173 return self._collectDir

174 175 name = property(_getName, _setName, None, "Name of the peer.") 176 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).") 177 178 179 ################# 180 # Public methods 181 ################# 182

183 - def stagePeer(self, targetDir, ownership=None, permissions=None):

184 """ 185 Stages data from the peer into the indicated local target directory. 186 187 The collect and target directories must both already exist before this 188 method is called. If passed in, ownership and permissions will be 189 applied to the files that are copied. 190 191 @note: The caller is responsible for checking that the indicator exists, 192 if they care. This function only stages the files within the directory. 193 194 @note: If you have user/group as strings, call the L{util.getUidGid} function 195 to get the associated uid/gid as an ownership tuple. 196 197 @param targetDir: Target directory to write data into 198 @type targetDir: String representing a directory on disk 199 200 @param ownership: Owner and group that the staged files should have 201 @type ownership: Tuple of numeric ids C{(uid, gid)} 202 203 @param permissions: Permissions that the staged files should have 204 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 205 206 @return: Number of files copied from the source directory to the target directory. 207 208 @raise ValueError: If collect directory is not a directory or does not exist 209 @raise ValueError: If target directory is not a directory, does not exist or is not absolute. 210 @raise ValueError: If a path cannot be encoded properly. 211 @raise IOError: If there were no files to stage (i.e. the directory was empty) 212 @raise IOError: If there is an IO error copying a file. 213 @raise OSError: If there is an OS error copying or changing permissions on a file 214 """ 215 targetDir = encodePath(targetDir) 216 if not os.path.isabs(targetDir): 217 logger.debug("Target directory [%s] not an absolute path." % targetDir) 218 raise ValueError("Target directory must be an absolute path.") 219 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir): 220 logger.debug("Collect directory [%s] is not a directory or does not exist on disk." % self.collectDir) 221 raise ValueError("Collect directory is not a directory or does not exist on disk.") 222 if not os.path.exists(targetDir) or not os.path.isdir(targetDir): 223 logger.debug("Target directory [%s] is not a directory or does not exist on disk." % targetDir) 224 raise ValueError("Target directory is not a directory or does not exist on disk.") 225 count = LocalPeer._copyLocalDir(self.collectDir, targetDir, ownership, permissions) 226 if count == 0: 227 raise IOError("Did not copy any files from local peer.") 228 return count

229

230 - def checkCollectIndicator(self, collectIndicator=None):

231 """ 232 Checks the collect indicator in the peer's staging directory. 233 234 When a peer has completed collecting its backup files, it will write an 235 empty indicator file into its collect directory. This method checks to 236 see whether that indicator has been written. We're "stupid" here - if 237 the collect directory doesn't exist, you'll naturally get back C{False}. 238 239 If you need to, you can override the name of the collect indicator file 240 by passing in a different name. 241 242 @param collectIndicator: Name of the collect indicator file to check 243 @type collectIndicator: String representing name of a file in the collect directory 244 245 @return: Boolean true/false depending on whether the indicator exists. 246 @raise ValueError: If a path cannot be encoded properly. 247 """ 248 collectIndicator = encodePath(collectIndicator) 249 if collectIndicator is None: 250 return os.path.exists(os.path.join(self.collectDir, DEF_COLLECT_INDICATOR)) 251 else: 252 return os.path.exists(os.path.join(self.collectDir, collectIndicator))

253

254 - def writeStageIndicator(self, stageIndicator=None, ownership=None, permissions=None):

255 """ 256 Writes the stage indicator in the peer's staging directory. 257 258 When the master has completed collecting its backup files, it will write 259 an empty indicator file into the peer's collect directory. The presence 260 of this file implies that the staging process is complete. 261 262 If you need to, you can override the name of the stage indicator file by 263 passing in a different name. 264 265 @note: If you have user/group as strings, call the L{util.getUidGid} 266 function to get the associated uid/gid as an ownership tuple. 267 268 @param stageIndicator: Name of the indicator file to write 269 @type stageIndicator: String representing name of a file in the collect directory 270 271 @param ownership: Owner and group that the indicator file should have 272 @type ownership: Tuple of numeric ids C{(uid, gid)} 273 274 @param permissions: Permissions that the indicator file should have 275 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 276 277 @raise ValueError: If collect directory is not a directory or does not exist 278 @raise ValueError: If a path cannot be encoded properly. 279 @raise IOError: If there is an IO error creating the file. 280 @raise OSError: If there is an OS error creating or changing permissions on the file 281 """ 282 stageIndicator = encodePath(stageIndicator) 283 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir): 284 logger.debug("Collect directory [%s] is not a directory or does not exist on disk." % self.collectDir) 285 raise ValueError("Collect directory is not a directory or does not exist on disk.") 286 if stageIndicator is None: 287 fileName = os.path.join(self.collectDir, DEF_STAGE_INDICATOR) 288 else: 289 fileName = os.path.join(self.collectDir, stageIndicator) 290 LocalPeer._copyLocalFile(None, fileName, ownership, permissions) # None for sourceFile results in an empty target

291 292 293 ################## 294 # Private methods 295 ################## 296

297 - def _copyLocalDir(sourceDir, targetDir, ownership=None, permissions=None):

298 """ 299 Copies files from the source directory to the target directory. 300 301 This function is not recursive. Only the files in the directory will be 302 copied. Ownership and permissions will be left at their default values 303 if new values are not specified. The source and target directories are 304 allowed to be soft links to a directory, but besides that soft links are 305 ignored. 306 307 @note: If you have user/group as strings, call the L{util.getUidGid} 308 function to get the associated uid/gid as an ownership tuple. 309 310 @param sourceDir: Source directory 311 @type sourceDir: String representing a directory on disk 312 313 @param targetDir: Target directory 314 @type targetDir: String representing a directory on disk 315 316 @param ownership: Owner and group that the copied files should have 317 @type ownership: Tuple of numeric ids C{(uid, gid)} 318 319 @param permissions: Permissions that the staged files should have 320 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 321 322 @return: Number of files copied from the source directory to the target directory. 323 324 @raise ValueError: If source or target is not a directory or does not exist. 325 @raise ValueError: If a path cannot be encoded properly. 326 @raise IOError: If there is an IO error copying the files. 327 @raise OSError: If there is an OS error copying or changing permissions on a files 328 """ 329 filesCopied = 0 330 sourceDir = encodePath(sourceDir) 331 targetDir = encodePath(targetDir) 332 for fileName in os.listdir(sourceDir): 333 sourceFile = os.path.join(sourceDir, fileName) 334 targetFile = os.path.join(targetDir, fileName) 335 LocalPeer._copyLocalFile(sourceFile, targetFile, ownership, permissions) 336 filesCopied += 1 337 return filesCopied

338 _copyLocalDir = staticmethod(_copyLocalDir) 339

340 - def _copyLocalFile(sourceFile=None, targetFile=None, ownership=None, permissions=None, overwrite=True):

341 """ 342 Copies a source file to a target file. 343 344 If the source file is C{None} then the target file will be created or 345 overwritten as an empty file. If the target file is C{None}, this method 346 is a no-op. Attempting to copy a soft link or a directory will result in 347 an exception. 348 349 @note: If you have user/group as strings, call the L{util.getUidGid} 350 function to get the associated uid/gid as an ownership tuple. 351 352 @note: We will not overwrite a target file that exists when this method 353 is invoked. If the target already exists, we'll raise an exception. 354 355 @param sourceFile: Source file to copy 356 @type sourceFile: String representing a file on disk, as an absolute path 357 358 @param targetFile: Target file to create 359 @type targetFile: String representing a file on disk, as an absolute path 360 361 @param ownership: Owner and group that the copied should have 362 @type ownership: Tuple of numeric ids C{(uid, gid)} 363 364 @param permissions: Permissions that the staged files should have 365 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 366 367 @param overwrite: Indicates whether it's OK to overwrite the target file. 368 @type overwrite: Boolean true/false. 369 370 @raise ValueError: If the passed-in source file is not a regular file. 371 @raise ValueError: If a path cannot be encoded properly. 372 @raise IOError: If the target file already exists. 373 @raise IOError: If there is an IO error copying the file 374 @raise OSError: If there is an OS error copying or changing permissions on a file 375 """ 376 targetFile = encodePath(targetFile) 377 sourceFile = encodePath(sourceFile) 378 if targetFile is None: 379 return 380 if not overwrite: 381 if os.path.exists(targetFile): 382 raise IOError("Target file [%s] already exists." % targetFile) 383 if sourceFile is None: 384 open(targetFile, "w").write("") 385 else: 386 if os.path.isfile(sourceFile) and not os.path.islink(sourceFile): 387 shutil.copy(sourceFile, targetFile) 388 else: 389 logger.debug("Source [%s] is not a regular file." % sourceFile) 390 raise ValueError("Source is not a regular file.") 391 if ownership is not None: 392 os.chown(targetFile, ownership[0], ownership[1]) 393 if permissions is not None: 394 os.chmod(targetFile, permissions)

395 _copyLocalFile = staticmethod(_copyLocalFile)

403 404 ###################### 405 # Class documentation 406 ###################### 407 408 """ 409 Backup peer representing a remote peer in a backup pool. 410 411 This is a class representing a remote (networked) peer in a backup pool. 412 Remote peers are backed up using an rcp-compatible copy command. A remote 413 peer has associated with it a name (which must be a valid hostname), a 414 collect directory, a working directory and a copy method (an rcp-compatible 415 command). 416 417 You can also set an optional local user value. This username will be used 418 as the local user for any remote copies that are required. It can only be 419 used if the root user is executing the backup. The root user will C{su} to 420 the local user and execute the remote copies as that user. 421 422 The copy method is associated with the peer and not with the actual request 423 to copy, because we can envision that each remote host might have a 424 different connect method. 425 426 The public methods other than the constructor are part of a "backup peer" 427 interface shared with the C{LocalPeer} class. 428 429 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator, 430 executeRemoteCommand, executeManagedAction, _getDirContents, 431 _copyRemoteDir, _copyRemoteFile, _pushLocalFile, name, collectDir, 432 remoteUser, rcpCommand, rshCommand, cbackCommand 433 """ 434 435 ############## 436 # Constructor 437 ############## 438

439 - def __init__(self, name, collectDir=None, workingDir=None, remoteUser=None, 440 rcpCommand=None, localUser=None, rshCommand=None, cbackCommand=None):

441 """ 442 Initializes a remote backup peer. 443 444 @note: If provided, each command will eventually be parsed into a list of 445 strings suitable for passing to C{util.executeCommand} in order to avoid 446 security holes related to shell interpolation. This parsing will be 447 done by the L{util.splitCommandLine} function. See the documentation for 448 that function for some important notes about its limitations. 449 450 @param name: Name of the backup peer 451 @type name: String, must be a valid DNS hostname 452 453 @param collectDir: Path to the peer's collect directory 454 @type collectDir: String representing an absolute path on the remote peer 455 456 @param workingDir: Working directory that can be used to create temporary files, etc. 457 @type workingDir: String representing an absolute path on the current host. 458 459 @param remoteUser: Name of the Cedar Backup user on the remote peer 460 @type remoteUser: String representing a username, valid via remote shell to the peer 461 462 @param localUser: Name of the Cedar Backup user on the current host 463 @type localUser: String representing a username, valid on the current host 464 465 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer 466 @type rcpCommand: String representing a system command including required arguments 467 468 @param rshCommand: An rsh-compatible copy command to use for remote shells to the peer 469 @type rshCommand: String representing a system command including required arguments 470 471 @param cbackCommand: A chack-compatible command to use for executing managed actions 472 @type cbackCommand: String representing a system command including required arguments 473 474 @raise ValueError: If collect directory is not an absolute path 475 """ 476 self._name = None 477 self._collectDir = None 478 self._workingDir = None 479 self._remoteUser = None 480 self._localUser = None 481 self._rcpCommand = None 482 self._rcpCommandList = None 483 self._rshCommand = None 484 self._rshCommandList = None 485 self._cbackCommand = None 486 self.name = name 487 self.collectDir = collectDir 488 self.workingDir = workingDir 489 self.remoteUser = remoteUser 490 self.localUser = localUser 491 self.rcpCommand = rcpCommand 492 self.rshCommand = rshCommand 493 self.cbackCommand = cbackCommand

494 495 496 ############# 497 # Properties 498 ############# 499

500 - def _setName(self, value):

501 """ 502 Property target used to set the peer name. 503 The value must be a non-empty string and cannot be C{None}. 504 @raise ValueError: If the value is an empty string or C{None}. 505 """ 506 if value is None or len(value) < 1: 507 raise ValueError("Peer name must be a non-empty string.") 508 self._name = value

509

510 - def _getName(self):

511 """ 512 Property target used to get the peer name. 513 """ 514 return self._name

515

516 - def _setCollectDir(self, value):

517 """ 518 Property target used to set the collect directory. 519 The value must be an absolute path and cannot be C{None}. 520 It does not have to exist on disk at the time of assignment. 521 @raise ValueError: If the value is C{None} or is not an absolute path. 522 @raise ValueError: If the value cannot be encoded properly. 523 """ 524 if value is not None: 525 if not os.path.isabs(value): 526 raise ValueError("Collect directory must be an absolute path.") 527 self._collectDir = encodePath(value)

528

529 - def _getCollectDir(self):

530 """ 531 Property target used to get the collect directory. 532 """ 533 return self._collectDir

534

535 - def _setWorkingDir(self, value):

536 """ 537 Property target used to set the working directory. 538 The value must be an absolute path and cannot be C{None}. 539 @raise ValueError: If the value is C{None} or is not an absolute path. 540 @raise ValueError: If the value cannot be encoded properly. 541 """ 542 if value is not None: 543 if not os.path.isabs(value): 544 raise ValueError("Working directory must be an absolute path.") 545 self._workingDir = encodePath(value)

546

547 - def _getWorkingDir(self):

548 """ 549 Property target used to get the working directory. 550 """ 551 return self._workingDir

552

553 - def _setRemoteUser(self, value):

554 """ 555 Property target used to set the remote user. 556 The value must be a non-empty string and cannot be C{None}. 557 @raise ValueError: If the value is an empty string or C{None}. 558 """ 559 if value is None or len(value) < 1: 560 raise ValueError("Peer remote user must be a non-empty string.") 561 self._remoteUser = value

562

563 - def _getRemoteUser(self):

564 """ 565 Property target used to get the remote user. 566 """ 567 return self._remoteUser

568

569 - def _setLocalUser(self, value):

570 """ 571 Property target used to set the local user. 572 The value must be a non-empty string if it is not C{None}. 573 @raise ValueError: If the value is an empty string. 574 """ 575 if value is not None: 576 if len(value) < 1: 577 raise ValueError("Peer local user must be a non-empty string.") 578 self._localUser = value

579

580 - def _getLocalUser(self):

581 """ 582 Property target used to get the local user. 583 """ 584 return self._localUser

585

586 - def _setRcpCommand(self, value):

587 """ 588 Property target to set the rcp command. 589 590 The value must be a non-empty string or C{None}. Its value is stored in 591 the two forms: "raw" as provided by the client, and "parsed" into a list 592 suitable for being passed to L{util.executeCommand} via 593 L{util.splitCommandLine}. 594 595 However, all the caller will ever see via the property is the actual 596 value they set (which includes seeing C{None}, even if we translate that 597 internally to C{DEF_RCP_COMMAND}). Internally, we should always use 598 C{self._rcpCommandList} if we want the actual command list. 599 600 @raise ValueError: If the value is an empty string. 601 """ 602 if value is None: 603 self._rcpCommand = None 604 self._rcpCommandList = DEF_RCP_COMMAND 605 else: 606 if len(value) >= 1: 607 self._rcpCommand = value 608 self._rcpCommandList = splitCommandLine(self._rcpCommand) 609 else: 610 raise ValueError("The rcp command must be a non-empty string.")

611

612 - def _getRcpCommand(self):

613 """ 614 Property target used to get the rcp command. 615 """ 616 return self._rcpCommand

617

618 - def _setRshCommand(self, value):

619 """ 620 Property target to set the rsh command. 621 622 The value must be a non-empty string or C{None}. Its value is stored in 623 the two forms: "raw" as provided by the client, and "parsed" into a list 624 suitable for being passed to L{util.executeCommand} via 625 L{util.splitCommandLine}. 626 627 However, all the caller will ever see via the property is the actual 628 value they set (which includes seeing C{None}, even if we translate that 629 internally to C{DEF_RSH_COMMAND}). Internally, we should always use 630 C{self._rshCommandList} if we want the actual command list. 631 632 @raise ValueError: If the value is an empty string. 633 """ 634 if value is None: 635 self._rshCommand = None 636 self._rshCommandList = DEF_RSH_COMMAND 637 else: 638 if len(value) >= 1: 639 self._rshCommand = value 640 self._rshCommandList = splitCommandLine(self._rshCommand) 641 else: 642 raise ValueError("The rsh command must be a non-empty string.")

643

644 - def _getRshCommand(self):

645 """ 646 Property target used to get the rsh command. 647 """ 648 return self._rshCommand

649

650 - def _setCbackCommand(self, value):

651 """ 652 Property target to set the cback command. 653 654 The value must be a non-empty string or C{None}. Unlike the other 655 command, this value is only stored in the "raw" form provided by the 656 client. 657 658 @raise ValueError: If the value is an empty string. 659 """ 660 if value is None: 661 self._cbackCommand = None 662 else: 663 if len(value) >= 1: 664 self._cbackCommand = value 665 else: 666 raise ValueError("The cback command must be a non-empty string.")

667

668 - def _getCbackCommand(self):

669 """ 670 Property target used to get the cback command. 671 """ 672 return self._cbackCommand

673 674 name = property(_getName, _setName, None, "Name of the peer (a valid DNS hostname).") 675 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).") 676 workingDir = property(_getWorkingDir, _setWorkingDir, None, "Path to the peer's working directory (an absolute local path).") 677 remoteUser = property(_getRemoteUser, _setRemoteUser, None, "Name of the Cedar Backup user on the remote peer.") 678 localUser = property(_getLocalUser, _setLocalUser, None, "Name of the Cedar Backup user on the current host.") 679 rcpCommand = property(_getRcpCommand, _setRcpCommand, None, "An rcp-compatible copy command to use for copying files.") 680 rshCommand = property(_getRshCommand, _setRshCommand, None, "An rsh-compatible command to use for remote shells to the peer.") 681 cbackCommand = property(_getCbackCommand, _setCbackCommand, None, "A chack-compatible command to use for executing managed actions.") 682 683 684 ################# 685 # Public methods 686 ################# 687

688 - def stagePeer(self, targetDir, ownership=None, permissions=None):

689 """ 690 Stages data from the peer into the indicated local target directory. 691 692 The target directory must already exist before this method is called. If 693 passed in, ownership and permissions will be applied to the files that 694 are copied. 695 696 @note: The returned count of copied files might be inaccurate if some of 697 the copied files already existed in the staging directory prior to the 698 copy taking place. We don't clear the staging directory first, because 699 some extension might also be using it. 700 701 @note: If you have user/group as strings, call the L{util.getUidGid} function 702 to get the associated uid/gid as an ownership tuple. 703 704 @note: Unlike the local peer version of this method, an I/O error might 705 or might not be raised if the directory is empty. Since we're using a 706 remote copy method, we just don't have the fine-grained control over our 707 exceptions that's available when we can look directly at the filesystem, 708 and we can't control whether the remote copy method thinks an empty 709 directory is an error. 710 711 @param targetDir: Target directory to write data into 712 @type targetDir: String representing a directory on disk 713 714 @param ownership: Owner and group that the staged files should have 715 @type ownership: Tuple of numeric ids C{(uid, gid)} 716 717 @param permissions: Permissions that the staged files should have 718 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 719 720 @return: Number of files copied from the source directory to the target directory. 721 722 @raise ValueError: If target directory is not a directory, does not exist or is not absolute. 723 @raise ValueError: If a path cannot be encoded properly. 724 @raise IOError: If there were no files to stage (i.e. the directory was empty) 725 @raise IOError: If there is an IO error copying a file. 726 @raise OSError: If there is an OS error copying or changing permissions on a file 727 """ 728 targetDir = encodePath(targetDir) 729 if not os.path.isabs(targetDir): 730 logger.debug("Target directory [%s] not an absolute path." % targetDir) 731 raise ValueError("Target directory must be an absolute path.") 732 if not os.path.exists(targetDir) or not os.path.isdir(targetDir): 733 logger.debug("Target directory [%s] is not a directory or does not exist on disk." % targetDir) 734 raise ValueError("Target directory is not a directory or does not exist on disk.") 735 count = RemotePeer._copyRemoteDir(self.remoteUser, self.localUser, self.name, 736 self._rcpCommand, self._rcpCommandList, 737 self.collectDir, targetDir, 738 ownership, permissions) 739 if count == 0: 740 raise IOError("Did not copy any files from local peer.") 741 return count

742

743 - def checkCollectIndicator(self, collectIndicator=None):

744 """ 745 Checks the collect indicator in the peer's staging directory. 746 747 When a peer has completed collecting its backup files, it will write an 748 empty indicator file into its collect directory. This method checks to 749 see whether that indicator has been written. If the remote copy command 750 fails, we return C{False} as if the file weren't there. 751 752 If you need to, you can override the name of the collect indicator file 753 by passing in a different name. 754 755 @note: Apparently, we can't count on all rcp-compatible implementations 756 to return sensible errors for some error conditions. As an example, the 757 C{scp} command in Debian 'woody' returns a zero (normal) status even when 758 it can't find a host or if the login or path is invalid. Because of 759 this, the implementation of this method is rather convoluted. 760 761 @param collectIndicator: Name of the collect indicator file to check 762 @type collectIndicator: String representing name of a file in the collect directory 763 764 @return: Boolean true/false depending on whether the indicator exists. 765 @raise ValueError: If a path cannot be encoded properly. 766 """ 767 try: 768 if collectIndicator is None: 769 sourceFile = os.path.join(self.collectDir, DEF_COLLECT_INDICATOR) 770 targetFile = os.path.join(self.workingDir, DEF_COLLECT_INDICATOR) 771 else: 772 collectIndicator = encodePath(collectIndicator) 773 sourceFile = os.path.join(self.collectDir, collectIndicator) 774 targetFile = os.path.join(self.workingDir, collectIndicator) 775 logger.debug("Fetch remote [%s] into [%s]." % (sourceFile, targetFile)) 776 if os.path.exists(targetFile): 777 try: 778 os.remove(targetFile) 779 except: 780 raise Exception("Error: collect indicator [%s] already exists!" % targetFile) 781 try: 782 RemotePeer._copyRemoteFile(self.remoteUser, self.localUser, self.name, 783 self._rcpCommand, self._rcpCommandList, 784 sourceFile, targetFile, 785 overwrite=False) 786 if os.path.exists(targetFile): 787 return True 788 else: 789 return False 790 except: 791 return False 792 finally: 793 if os.path.exists(targetFile): 794 try: 795 os.remove(targetFile) 796 except: pass

797

798 - def writeStageIndicator(self, stageIndicator=None):

799 """ 800 Writes the stage indicator in the peer's staging directory. 801 802 When the master has completed collecting its backup files, it will write 803 an empty indicator file into the peer's collect directory. The presence 804 of this file implies that the staging process is complete. 805 806 If you need to, you can override the name of the stage indicator file by 807 passing in a different name. 808 809 @note: If you have user/group as strings, call the L{util.getUidGid} function 810 to get the associated uid/gid as an ownership tuple. 811 812 @param stageIndicator: Name of the indicator file to write 813 @type stageIndicator: String representing name of a file in the collect directory 814 815 @raise ValueError: If a path cannot be encoded properly. 816 @raise IOError: If there is an IO error creating the file. 817 @raise OSError: If there is an OS error creating or changing permissions on the file 818 """ 819 stageIndicator = encodePath(stageIndicator) 820 if stageIndicator is None: 821 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR) 822 targetFile = os.path.join(self.collectDir, DEF_STAGE_INDICATOR) 823 else: 824 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR) 825 targetFile = os.path.join(self.collectDir, stageIndicator) 826 try: 827 if not os.path.exists(sourceFile): 828 open(sourceFile, "w").write("") 829 RemotePeer._pushLocalFile(self.remoteUser, self.localUser, self.name, 830 self._rcpCommand, self._rcpCommandList, 831 sourceFile, targetFile) 832 finally: 833 if os.path.exists(sourceFile): 834 try: 835 os.remove(sourceFile) 836 except: pass

837

838 - def executeRemoteCommand(self, command):

839 """ 840 Executes a command on the peer via remote shell. 841 842 @param command: Command to execute 843 @type command: String command-line suitable for use with rsh. 844 845 @raise IOError: If there is an error executing the command on the remote peer. 846 """ 847 RemotePeer._executeRemoteCommand(self.remoteUser, self.localUser, 848 self.name, self._rshCommand, 849 self._rshCommandList, command)

850

851 - def executeManagedAction(self, action, fullBackup):

852 """ 853 Executes a managed action on this peer. 854 855 @param action: Name of the action to execute. 856 @param fullBackup: Whether a full backup should be executed. 857 858 @raise IOError: If there is an error executing the action on the remote peer. 859 """ 860 try: 861 command = RemotePeer._buildCbackCommand(self.cbackCommand, action, fullBackup) 862 self.executeRemoteCommand(command) 863 except IOError, e: 864 logger.info(e) 865 raise IOError("Failed to execute action [%s] on managed client [%s]." % (action, self.name))

866 867 868 ################## 869 # Private methods 870 ################## 871

872 - def _getDirContents(path):

873 """ 874 Returns the contents of a directory in terms of a Set. 875 876 The directory's contents are read as a L{FilesystemList} containing only 877 files, and then the list is converted into a C{sets.Set} object for later 878 use. 879 880 @param path: Directory path to get contents for 881 @type path: String representing a path on disk 882 883 @return: Set of files in the directory 884 @raise ValueError: If path is not a directory or does not exist. 885 """ 886 contents = FilesystemList() 887 contents.excludeDirs = True 888 contents.excludeLinks = True 889 contents.addDirContents(path) 890 return sets.Set(contents)

891 _getDirContents = staticmethod(_getDirContents) 892

893 - def _copyRemoteDir(remoteUser, localUser, remoteHost, rcpCommand, rcpCommandList, 894 sourceDir, targetDir, ownership=None, permissions=None):

895 """ 896 Copies files from the source directory to the target directory. 897 898 This function is not recursive. Only the files in the directory will be 899 copied. Ownership and permissions will be left at their default values 900 if new values are not specified. Behavior when copying soft links from 901 the collect directory is dependent on the behavior of the specified rcp 902 command. 903 904 @note: The returned count of copied files might be inaccurate if some of 905 the copied files already existed in the staging directory prior to the 906 copy taking place. We don't clear the staging directory first, because 907 some extension might also be using it. 908 909 @note: If you have user/group as strings, call the L{util.getUidGid} function 910 to get the associated uid/gid as an ownership tuple. 911 912 @note: We don't have a good way of knowing exactly what files we copied 913 down from the remote peer, unless we want to parse the output of the rcp 914 command (ugh). We could change permissions on everything in the target 915 directory, but that's kind of ugly too. Instead, we use Python's set 916 functionality to figure out what files were added while we executed the 917 rcp command. This isn't perfect - for instance, it's not correct if 918 someone else is messing with the directory at the same time we're doing 919 the remote copy - but it's about as good as we're going to get. 920 921 @note: Apparently, we can't count on all rcp-compatible implementations 922 to return sensible errors for some error conditions. As an example, the 923 C{scp} command in Debian 'woody' returns a zero (normal) status even 924 when it can't find a host or if the login or path is invalid. We try 925 to work around this by issuing C{IOError} if we don't copy any files from 926 the remote host. 927 928 @param remoteUser: Name of the Cedar Backup user on the remote peer 929 @type remoteUser: String representing a username, valid via the copy command 930 931 @param localUser: Name of the Cedar Backup user on the current host 932 @type localUser: String representing a username, valid on the current host 933 934 @param remoteHost: Hostname of the remote peer 935 @type remoteHost: String representing a hostname, accessible via the copy command 936 937 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer 938 @type rcpCommand: String representing a system command including required arguments 939 940 @param rcpCommandList: An rcp-compatible copy command to use for copying files 941 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand} 942 943 @param sourceDir: Source directory 944 @type sourceDir: String representing a directory on disk 945 946 @param targetDir: Target directory 947 @type targetDir: String representing a directory on disk 948 949 @param ownership: Owner and group that the copied files should have 950 @type ownership: Tuple of numeric ids C{(uid, gid)} 951 952 @param permissions: Permissions that the staged files should have 953 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 954 955 @return: Number of files copied from the source directory to the target directory. 956 957 @raise ValueError: If source or target is not a directory or does not exist. 958 @raise IOError: If there is an IO error copying the files. 959 """ 960 beforeSet = RemotePeer._getDirContents(targetDir) 961 if localUser is not None: 962 try: 963 if os.getuid() != 0: 964 raise IOError("Only root can remote copy as another user.") 965 except AttributeError: pass 966 actualCommand = "%s %s@%s:%s/* %s" % (rcpCommand, remoteUser, remoteHost, sourceDir, targetDir) 967 command = resolveCommand(SU_COMMAND) 968 result = executeCommand(command, [localUser, "-c", actualCommand])[0] 969 if result != 0: 970 raise IOError("Error (%d) copying files from remote host as local user [%s]." % (result, localUser)) 971 else: 972 copySource = "%s@%s:%s/*" % (remoteUser, remoteHost, sourceDir) 973 command = resolveCommand(rcpCommandList) 974 result = executeCommand(command, [copySource, targetDir])[0] 975 if result != 0: 976 raise IOError("Error (%d) copying files from remote host (using no local user)." % result) 977 afterSet = RemotePeer._getDirContents(targetDir) 978 if len(afterSet) == 0: 979 raise IOError("Did not copy any files from remote peer.") 980 differenceSet = afterSet.difference(beforeSet) # files we added as part of copy 981 if len(differenceSet) == 0: 982 raise IOError("Apparently did not copy any new files from remote peer.") 983 for targetFile in differenceSet: 984 if ownership is not None: 985 os.chown(targetFile, ownership[0], ownership[1]) 986 if permissions is not None: 987 os.chmod(targetFile, permissions) 988 return len(differenceSet)

989 _copyRemoteDir = staticmethod(_copyRemoteDir) 990

991 - def _copyRemoteFile(remoteUser, localUser, remoteHost, 992 rcpCommand, rcpCommandList, 993 sourceFile, targetFile, ownership=None, 994 permissions=None, overwrite=True):

995 """ 996 Copies a remote source file to a target file. 997 998 @note: Internally, we have to go through and escape any spaces in the 999 source path with double-backslash, otherwise things get screwed up. It 1000 doesn't seem to be required in the target path. I hope this is portable 1001 to various different rcp methods, but I guess it might not be (all I have 1002 to test with is OpenSSH). 1003 1004 @note: If you have user/group as strings, call the L{util.getUidGid} function 1005 to get the associated uid/gid as an ownership tuple. 1006 1007 @note: We will not overwrite a target file that exists when this method 1008 is invoked. If the target already exists, we'll raise an exception. 1009 1010 @note: Apparently, we can't count on all rcp-compatible implementations 1011 to return sensible errors for some error conditions. As an example, the 1012 C{scp} command in Debian 'woody' returns a zero (normal) status even when 1013 it can't find a host or if the login or path is invalid. We try to work 1014 around this by issuing C{IOError} the target file does not exist when 1015 we're done. 1016 1017 @param remoteUser: Name of the Cedar Backup user on the remote peer 1018 @type remoteUser: String representing a username, valid via the copy command 1019 1020 @param remoteHost: Hostname of the remote peer 1021 @type remoteHost: String representing a hostname, accessible via the copy command 1022 1023 @param localUser: Name of the Cedar Backup user on the current host 1024 @type localUser: String representing a username, valid on the current host 1025 1026 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer 1027 @type rcpCommand: String representing a system command including required arguments 1028 1029 @param rcpCommandList: An rcp-compatible copy command to use for copying files 1030 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand} 1031 1032 @param sourceFile: Source file to copy 1033 @type sourceFile: String representing a file on disk, as an absolute path 1034 1035 @param targetFile: Target file to create 1036 @type targetFile: String representing a file on disk, as an absolute path 1037 1038 @param ownership: Owner and group that the copied should have 1039 @type ownership: Tuple of numeric ids C{(uid, gid)} 1040 1041 @param permissions: Permissions that the staged files should have 1042 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 1043 1044 @param overwrite: Indicates whether it's OK to overwrite the target file. 1045 @type overwrite: Boolean true/false. 1046 1047 @raise IOError: If the target file already exists. 1048 @raise IOError: If there is an IO error copying the file 1049 @raise OSError: If there is an OS error changing permissions on the file 1050 """ 1051 if not overwrite: 1052 if os.path.exists(targetFile): 1053 raise IOError("Target file [%s] already exists." % targetFile) 1054 if localUser is not None: 1055 try: 1056 if os.getuid() != 0: 1057 raise IOError("Only root can remote copy as another user.") 1058 except AttributeError: pass 1059 actualCommand = "%s %s@%s:%s %s" % (rcpCommand, remoteUser, remoteHost, sourceFile.replace(" ", "\\ "), targetFile) 1060 command = resolveCommand(SU_COMMAND) 1061 result = executeCommand(command, [localUser, "-c", actualCommand])[0] 1062 if result != 0: 1063 raise IOError("Error (%d) copying [%s] from remote host as local user [%s]." % (result, sourceFile, localUser)) 1064 else: 1065 copySource = "%s@%s:%s" % (remoteUser, remoteHost, sourceFile.replace(" ", "\\ ")) 1066 command = resolveCommand(rcpCommandList) 1067 result = executeCommand(command, [copySource, targetFile])[0] 1068 if result != 0: 1069 raise IOError("Error (%d) copying [%s] from remote host (using no local user)." % (result, sourceFile)) 1070 if not os.path.exists(targetFile): 1071 raise IOError("Apparently unable to copy file from remote host.") 1072 if ownership is not None: 1073 os.chown(targetFile, ownership[0], ownership[1]) 1074 if permissions is not None: 1075 os.chmod(targetFile, permissions)

1076 _copyRemoteFile = staticmethod(_copyRemoteFile) 1077

1078 - def _pushLocalFile(remoteUser, localUser, remoteHost, 1079 rcpCommand, rcpCommandList, 1080 sourceFile, targetFile, overwrite=True):

1081 """ 1082 Copies a local source file to a remote host. 1083 1084 @note: We will not overwrite a target file that exists when this method 1085 is invoked. If the target already exists, we'll raise an exception. 1086 1087 @note: Internally, we have to go through and escape any spaces in the 1088 source and target paths with double-backslash, otherwise things get 1089 screwed up. I hope this is portable to various different rcp methods, 1090 but I guess it might not be (all I have to test with is OpenSSH). 1091 1092 @note: If you have user/group as strings, call the L{util.getUidGid} function 1093 to get the associated uid/gid as an ownership tuple. 1094 1095 @param remoteUser: Name of the Cedar Backup user on the remote peer 1096 @type remoteUser: String representing a username, valid via the copy command 1097 1098 @param localUser: Name of the Cedar Backup user on the current host 1099 @type localUser: String representing a username, valid on the current host 1100 1101 @param remoteHost: Hostname of the remote peer 1102 @type remoteHost: String representing a hostname, accessible via the copy command 1103 1104 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer 1105 @type rcpCommand: String representing a system command including required arguments 1106 1107 @param rcpCommandList: An rcp-compatible copy command to use for copying files 1108 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand} 1109 1110 @param sourceFile: Source file to copy 1111 @type sourceFile: String representing a file on disk, as an absolute path 1112 1113 @param targetFile: Target file to create 1114 @type targetFile: String representing a file on disk, as an absolute path 1115 1116 @param overwrite: Indicates whether it's OK to overwrite the target file. 1117 @type overwrite: Boolean true/false. 1118 1119 @raise IOError: If there is an IO error copying the file 1120 @raise OSError: If there is an OS error changing permissions on the file 1121 """ 1122 if not overwrite: 1123 if os.path.exists(targetFile): 1124 raise IOError("Target file [%s] already exists." % targetFile) 1125 if localUser is not None: 1126 try: 1127 if os.getuid() != 0: 1128 raise IOError("Only root can remote copy as another user.") 1129 except AttributeError: pass 1130 actualCommand = '%s "%s" "%s@%s:%s"' % (rcpCommand, sourceFile, remoteUser, remoteHost, targetFile) 1131 command = resolveCommand(SU_COMMAND) 1132 result = executeCommand(command, [localUser, "-c", actualCommand])[0] 1133 if result != 0: 1134 raise IOError("Error (%d) copying [%s] to remote host as local user [%s]." % (result, sourceFile, localUser)) 1135 else: 1136 copyTarget = "%s@%s:%s" % (remoteUser, remoteHost, targetFile.replace(" ", "\\ ")) 1137 command = resolveCommand(rcpCommandList) 1138 result = executeCommand(command, [sourceFile.replace(" ", "\\ "), copyTarget])[0] 1139 if result != 0: 1140 raise IOError("Error (%d) copying [%s] to remote host (using no local user)." % (result, sourceFile))

1141 _pushLocalFile = staticmethod(_pushLocalFile) 1142

1143 - def _executeRemoteCommand(remoteUser, localUser, remoteHost, rshCommand, rshCommandList, remoteCommand):

1144 """ 1145 Executes a command on the peer via remote shell. 1146 1147 @param remoteUser: Name of the Cedar Backup user on the remote peer 1148 @type remoteUser: String representing a username, valid on the remote host 1149 1150 @param localUser: Name of the Cedar Backup user on the current host 1151 @type localUser: String representing a username, valid on the current host 1152 1153 @param remoteHost: Hostname of the remote peer 1154 @type remoteHost: String representing a hostname, accessible via the copy command 1155 1156 @param rshCommand: An rsh-compatible copy command to use for remote shells to the peer 1157 @type rshCommand: String representing a system command including required arguments 1158 1159 @param rshCommandList: An rsh-compatible copy command to use for remote shells to the peer 1160 @type rshCommandList: Command as a list to be passed to L{util.executeCommand} 1161 1162 @param remoteCommand: The command to be executed on the remote host 1163 @type remoteCommand: String command-line, with no special shell characters ($, <, etc.) 1164 1165 @raise IOError: If there is an error executing the remote command 1166 """ 1167 actualCommand = "%s %s@%s '%s'" % (rshCommand, remoteUser, remoteHost, remoteCommand) 1168 if localUser is not None: 1169 try: 1170 if os.getuid() != 0: 1171 raise IOError("Only root can remote shell as another user.") 1172 except AttributeError: pass 1173 command = resolveCommand(SU_COMMAND) 1174 result = executeCommand(command, [localUser, "-c", actualCommand])[0] 1175 if result != 0: 1176 raise IOError("Command failed [su -c %s \"%s\"]" % (localUser, actualCommand)) 1177 else: 1178 command = resolveCommand(rshCommandList) 1179 result = executeCommand(command, ["%s@%s" % (remoteUser, remoteHost), "%s" % remoteCommand])[0] 1180 if result != 0: 1181 raise IOError("Command failed [%s]" % (actualCommand))

1182 _executeRemoteCommand = staticmethod(_executeRemoteCommand) 1183

1184 - def _buildCbackCommand(cbackCommand, action, fullBackup):

1185 """ 1186 Builds a Cedar Backup command line for the named action. 1187 1188 @note: If the cback command is None, then DEF_CBACK_COMMAND is used. 1189 1190 @param cbackCommand: cback command to execute, including required options 1191 @param action: Name of the action to execute. 1192 @param fullBackup: Whether a full backup should be executed. 1193 1194 @return: String suitable for passing to L{_executeRemoteCommand} as remoteCommand. 1195 @raise ValueError: If action is None. 1196 """ 1197 if action is None: 1198 raise ValueError("Action cannot be None.") 1199 if cbackCommand is None: 1200 cbackCommand = DEF_CBACK_COMMAND 1201 if fullBackup: 1202 return "%s --full %s" % (cbackCommand, action) 1203 else: 1204 return "%s %s" % (cbackCommand, action)

1205 _buildCbackCommand = staticmethod(_buildCbackCommand)

Source Code for Module CedarBackup2.peer