ftpUtility.py

#!/usr/bin/env python

import copy
import os
import time
from ftplib import FTP
from dm.common.utility.timeUtility import TimeUtility
from dm.common.utility.loggingManager import LoggingManager
import urlparse

class FtpUtility:

    def __init__(self, host, port, username=None, password=None):
        self.host = host
        self.port = port
        self.username = username
        self.password = password
        self.ftpClient = None
        self.mlsdFileStatDict = {}

    @classmethod
    def parseFtpUrl(cls, url, defaultHost=None, defaultPort=None):
        host = defaultHost
        port = defaultPort
        scheme = None
        dirPath = url
        if url.startswith('ftp://'):
            parseResult = urlparse.urlparse(url)
            scheme = parseResult.scheme
            netlocTokens = parseResult.netloc.split(':')
            host = netlocTokens[0]
            if len(netlocTokens) > 1:
                port = int(netlocTokens[1])
            dirPath = parseResult.path
        return (scheme, host, port, dirPath)

    @classmethod
    def getFtpClient(cls, host, port, username=None, password=None):
        ftp = FTP()
        ftp.connect(host, port)
        ftp.login(username, password)
        return ftp

    @classmethod
    def getLogger(cls):
        logger = LoggingManager.getInstance().getLogger(cls.__name__)
        return logger


    def __parseKeyValue(cls, keyValue, outputDict={}):
        key,value = keyValue.split('=')
        value = value.strip()
        outputDict[key] = value
        return outputDict
        
    def __parseMlsdOutput(self, line):
        # ['Type=dir', 'Modify=20151018024430', 'Size=4096', 'Perm=el', 'UNIX.mode=0775', 'UNIX.owner=sveseli', 'UNIX.uid=500', 'UNIX.group=sveseli', 'UNIX.gid=500', 'Unique=fd00-c2e3e', ' dir2\r']
        parts = line.split(';')
        parseDict = {}
        self.__parseKeyValue(parts[0], parseDict)
        self.__parseKeyValue(parts[1], parseDict)
        self.__parseKeyValue(parts[2], parseDict)
        name = parts[-1].strip()
        parseDict['Name'] = name
        type = parseDict.get('Type', '')
        if type == 'dir' :
            self.mlsdDirList.append(name)
        elif type == 'file':
            self.mlsdFileDict[name] = parseDict

    def __parseMlsdFileStat(self, line):
        # ['Type=dir', 'Modify=20151018024430', 'Size=4096', 'Perm=el', 'UNIX.mode=0775', 'UNIX.owner=sveseli', 'UNIX.uid=500', 'UNIX.group=sveseli', 'UNIX.gid=500', 'Unique=fd00-c2e3e', ' dir2\r']
        parts = line.split(';')
        parseDict = {}
        self.__parseKeyValue(parts[0], parseDict)
        self.__parseKeyValue(parts[1], parseDict)
        self.__parseKeyValue(parts[2], parseDict)
        name = parts[-1].strip()
        self.mlsdFileStatDict[name] = parseDict

    def getFiles(self, dir, fileDict={}):
        if not self.ftpClient:
            self.ftpClient = self.getFtpClient(self.host, self.port, self.username, self.password)
        # Need these to be class members for the callback function
        self.mlsdFileDict = {}
        self.mlsdDirList = []
        self.ftpClient.retrlines('MLSD %s' % dir, self.__parseMlsdOutput)
        for (fileName,fileInfo) in self.mlsdFileDict.items():
            filePath = '%s/%s' % (dir, fileName)
            fileDict[filePath] = fileInfo
        for d in copy.copy(self.mlsdDirList):
            dirPath = '%s/%s' % (dir,d)
            self.getFiles(dirPath,fileDict)
        return fileDict

    def getMd5Sum(self, filePath, fileInfo={}):
        if not self.ftpClient:
            self.ftpClient = self.getFtpClient(self.host, self.port, self.username, self.password)
        md5Sum = self.ftpClient.sendcmd('CKSM MD5 0 -1 %s' % filePath).split()[-1]
        fileInfo['md5Sum'] = md5Sum
        return md5Sum

    def statFile(self, filePath, fileInfo={}):
        fileName = os.path.basename(filePath)
        if not self.ftpClient:
            self.ftpClient = self.getFtpClient(self.host, self.port, self.username, self.password)
        # Need this to be class members for the callback function
        self.ftpClient.retrlines('MLSD %s' % filePath, self.__parseMlsdFileStat)
        fileStatDict = self.mlsdFileStatDict.get(fileName)
        if fileStatDict:
            fileInfo['fileSize'] = fileStatDict.get('Size')
            modifyTime = fileStatDict.get('Modify')
            modifyTime = time.mktime(time.strptime(modifyTime, '%Y%m%d%H%M%S'))
            fileInfo['fileModificationTime'] = modifyTime
            fileInfo['fileModificationTimeStamp'] = TimeUtility.formatLocalTimeStamp(modifyTime)
            del self.mlsdFileStatDict[fileName]
        return fileInfo

#######################################################################
# Testing.

if __name__ == '__main__':
    ftpUtility = FtpUtility('s8dserv', 2811)
    files = ftpUtility.getFiles('/export/8-id-i/test')
    print files
    print ftpUtility.getMd5Sum('/export/8-id-i/test/testfile01')
    print ftpUtility.statFile('/export/8-id-i/test/testfile01')