# -*- coding: iso-8859-1 -*- # ----------------------------------------------------------------------- __file__ = "gui_ftp.py" __doc__= "FTP functions for ul/downloading one or several server in parallele with multiple connections." __author__ = "Philippe Blanquier" __version__ = "v0.1" __date__ = "24/06/2011" __copyright__ = "Alcatel-Lucent" __contact__ = "philippe.blanquier@alcatel-lucent.com" # ----------------------------------------------------------------------- # Python include import os,sys,ftplib,glob,time,threading,socket,copy # Private include import gui_common,gui_scenario_manager,gui_user ### ------------ ### ### Private data ### ### ------------ ### _ftps_certificate_file_name = "C:"+os.sep+"Nemo_Pyc"+os.sep+"sftp_certificate.crt" #: FTPS certificate for labo servers _ftp_error_tag = "FTP" _ftp_block_size = 64*gui_common.kilo_unit #: Maximum chunk size to read/write on the low-level socket object created to do the actual transfer _ftp_repeat_nb = 10 #: Maximal command repeatition number before saying 'KO'... _ftp_repeat_waiting_delay = 30.0 #: Delay between 2 repetition when a failure is detected _upload_cmd = "Upload" _download_cmd = "Download" _file_size_tag = "File size" _ftp_posix_separator = '/' #: Normalized FTP directory separator ( UNIX POSIX !) _ftp_dir_size = -1 #: Dummy FTP directory size _ftp_create_dir_mutex = gui_common.nice_mutex( this_name = "FTP_Dir_Mutex") #: Internal directory creation protection # see: http://en.wikipedia.org/wiki/List_of_FTP_server_return_codes # # FTP server return codes always have three digits, and each digit has a special meaning. # # The first digit denotes whether the response is good, bad or incomplete: # 1xx Positive Preliminary reply # 2xx Positive Completion reply # 3xx Positive Intermediate reply # 4xx Transient Negative Completion reply # 5xx Permanent Negative Completion reply # 6xx Protected reply # The second digit is a grouping digit and encodes the following information: # x0x Syntax # x1x Information # x2x Connections # x3x Authentication and accounting # x4x Unspecified as of RFC 959. # x5x File system ### The requested action is being taken. Expect a reply before proceeding with a new command. # 100 Series: The requested action is being initiated, expect another reply before proceeding with a new command. # 110 Restart marker replay . In this case, the text is exact and not left to the particular implementation; # it must read: MARK yyyy = mmmm where yyyy is User-process data stream marker, and mmmm server's equivalent marker (note the spaces between markers and "="). # 120 Service ready in nnn minutes. # 125 Data connection already open; transfer starting. # 150 File status okay; about to open data connection. ### The requested action has been successfully completed. # 200 Command okay. # 202 Command not implemented, superfluous at this site. # 211 System status, or system help reply. # 212 Directory status. # 213 File status. # 214 Help message.On how to use the server or the meaning of a particular non-standard command. This reply is useful only to the human user. # 215 NAME system type. Where NAME is an official system name from the registry kept by IANA. # 220 Service ready for new user. # 221 Service closing control connection. # 225 Data connection open; no transfer in progress. # 226 Closing data connection. Requested file action successful (for example, file transfer or file abort). # 227 Entering Passive Mode (h1,h2,h3,h4,p1,p2). # 228 Entering Long Passive Mode (long address, port). # 229 Entering Extended Passive Mode (|||port|). # 230 User logged in, proceed. Logged out if appropriate. # 231 User logged out; service terminated. # 232 Logout command noted, will complete when transfer done. # 250 Requested file action okay, completed. # 257 "PATHNAME" created. ### The command has been accepted, but the requested action is being held pending receipt of further information. # 331 User name okay, need password. # 332 Need account for login. # 350 Requested file action pending further information ### <<>> 400...499 ### The command was not accepted and the requested action did not take place. ### The error condition is temporary, however, and the action may be requested again. # 421 Service not available, closing control connection. # 421 Connection timed out # 425 Can't open data connection. # 426 Connection closed; transfer aborted. # 430 Invalid username or password # 434 Requested host unavailable. # 450 Requested file action not taken. # 451 Requested action aborted. Local error in processing. # 452 Requested action not taken. Insufficient storage space in system.File unavailable (e.g., file busy). ### <<>> 500...599 ### The command was not accepted and the requested action did not take place. # 500 Syntax error, command unrecognized. This may include errors such as command line too long. # 501 Syntax error in parameters or arguments. # 502 Command not implemented. # 503 Bad sequence of commands. # 504 Command not implemented for that parameter. # 530 Not logged in. # 532 Need account for storing files. # 550 Requested action not taken. File unavailable (e.g., file not found, no access). # 551 Requested action aborted. Page type unknown. # 552 Requested file action aborted. Exceeded storage allocation (for current directory or dataset). # 553 Requested action not taken. File name not allowed. # 631 Integrity protected reply. # 632 Confidentiality and integrity protected reply. # 633 Confidentiality protected reply. ### Common Winsock Error Codes (full error: http://kb.globalscape.com/KnowledgebaseArticle10140.aspx) #10054 Connection reset by peer. The connection was forcibly closed by the remote host. (Informational) #10060 Cannot connect to remote server. Generally a time-out error. Try switching from PASV to PORT mode, or try increasing the time-out value. #10061 Cannot connect to remote server. The connection is actively refused by the server. Try switching the connection port. #10066 Directory not empty. The server will not delete this directory while there are files/folders in it. If you want to remove the directory, first archive or delete the files in it. #10068 Too many users, server is full. Try logging in at another time. # The following are the FTP commands (see RFC 959): # ABOR # ACCT # ALLO # [ R ] # APPE # CDUP # CWD # DELE # HELP [ ] # LIST [ ] # MODE # MKD # NLST [ ] # NOOP # PASS # PASV # PORT # PWD # QUIT # REIN # REST # RETR # RMD # RNFR # RNTO # SITE # SMNT # STAT [ ] # STOR # STOU # STRU # SYST # TYPE # USER ### ----------------- ### ### Private functions ### ### ----------------- ### # ------------------------------------------ # Private Function : _ftp_close_connection() # ------------------------------------------ def _ftp_close_connection( this_ftp_id = None): """ Close the FTP connection. @param this_ftp_id: FTP connection ident @type this_ftp_id: ftplib object @note: - use 'quit()'command first: this is the polite way to close a connection - if 'quit()' fails, use 'close()' command @return: None @see: https://docs.python.org/release/2.6.9/library/ftplib.html """ if this_ftp_id is not None: # Stop current transfert (if any) #try: # this_ftp_id.abort() #except: # pass # Be polite with the remote host to quit and close the connection. try: this_ftp_id.quit() except: # Oops ! Close the connection unilaterally ! try: this_ftp_id.close() except: # Ignore the error pass return # -------------------------------------------- # Private Function : _ftp_get_max_connection() # -------------------------------------------- def _ftp_get_max_connection( this_address = "0.0.0.0"): """ Determine the maximal connection in regard of the remote server adress IP. - 4 for intranet or VPN to Labo server: speed up for end user - 1 for other situation @param this_address: server IP address @type this_address: ascii string @note: the values MUST be compatible with the Filezilla server preferences running on the remote servers ! """ if gui_common.valid_labo_ip_addr( this_ip_addr = this_address , this_all_server = True ) is True: answer = 4 else: # Unknown answer = 1 return answer # --------------------- # _ftp_repeat_command() # --------------------- def _ftp_repeat_command( this_err_msg = None): """ Try to detect FTP errors whose commands could be repeated (i.e: 'beyond help'). Some errors are only taken into account at present for repetitions: - 421 User limit reached. - 421 Max connections reached. - 421 Too many users are connected, please try again later. - 421 Connection timed out - 425 Can't open data connection (*) - 426 Connection closed; transfer aborted (*) - 450 Requested file action not taken. - 451 Requested action aborted. Local error in processing. - 550 can't access file. (on SFTP 'STOR' command) (*) - SSL: DECRYPTION_FAILED_OR_BAD_RECORD_MAC - EOF occurred in violation of protocol @param this_err_msg: full FTP error @type this_err_msg: ascii string @return: True = repeat allowed, False = no repeatition @rtype: boolean @note: The errors (*) 425 & 426 could occure during upload... """ if this_err_msg is None: return False beyond_help_error_list = [ "User limit reached" , "Max connections reached" , "Too many users are connected" , "Connection timed out" , _file_size_tag # <<>> # [Errno 10013] An attempt was made to access a socket in a way forbidden by its access permissions. , "Errno 10013" # [Errno 10060] A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond , "Errno 10060" # "426 Connection closed; transfer aborted. May occure during transfert. Try to work around using activ/passif mode... see: http://support.isotools.fr/ticket/Customer/KBArticle.aspx?articleid=217 , "transfer aborted" # 425 Can't open data connection , "Can't open data connection" # 450 Requested file action not taken. , "Requested file action not taken" # 450 TLS session of data connection has not resumed or the session does not match the control connection , "TLS session of data connection has not resumed" # 451 Requested action aborted. Local error in processing. , "Local error in processing" # 550 can't access file. , "can't access file" # SSL errors , "DECRYPTION_FAILED_OR_BAD_RECORD_MAC" , "EOF occurred in violation of protocol" ] answer = False for code_idx in beyond_help_error_list: if code_idx in this_err_msg: # Found ! answer = True break # Next code_idx continue return answer # -------------------------- # Private Class : _ftp_mutex # -------------------------- class _ftp_mutex(gui_common.Singleton): """ Manage only one acces to each remote host when a remote directory has to be created """ ip_mutex_dico = {} # --------------------- # _ftp_mutex.__init__() # --------------------- def __init__( self , this_ip_addr ): if _ftp_mutex.ip_mutex_dico.get(this_ip_addr, None) is None: _ftp_mutex.ip_mutex_dico[this_ip_addr] = gui_common.nice_mutex( this_name = "FTP_Dir_Mutex_%s"%(this_ip_addr)) #: Internal directory creation protection return # ------------------ # _ftp_mutex.error() # ------------------ def error( self , this_msg ): """ Shortcut access to display an error @param this_msg: input message describing the error @type this_msg: ascii string @return: None """ gui_common.save_error( this_msg = this_msg , this_object_class = self , this_file_name = __file__ , this_mail_subject = _ftp_error_tag+" Mutex" ) # ---------------- # _ftp_mutex.get() # ---------------- def get( self , this_ip_addr ): local_mutex = _ftp_mutex.ip_mutex_dico.get(this_ip_addr, None) if local_mutex is None: # Oops ! self.error( this_msg = "No mutex for %s"%(this_ip_addr)) else: local_mutex.get() return # -------------------- # _ftp_mutex.release() # -------------------- def release( self , this_ip_addr ): local_mutex = _ftp_mutex.ip_mutex_dico.get(this_ip_addr, None) if local_mutex is None: # Oops ! self.error( this_msg = "No mutex for %s"%(this_ip_addr)) else: local_mutex.release() return # ---------------------------------- # Private Class : _ftp_slave_thread # --------------------------------- class _ftp_slave_thread( threading.Thread): """ Independant FPT thread to upload or download in parallele on a dedicated server. @note: A file size check is done to prevent a 'null file size' on the tranfered file (ftplib bug ?) @note: see: http://support.microsoft.com/kb/466868/fr @sort: _*,a*,b*,c*,d*,e*,f*,g*,h*,i*,j*,k*,l*,m*,n*,o*,p*,q*,r*,s*,t*,u*,v*,w*,x*,y*,z* """ # ---------------------------- # _ftp_slave_thread.__init__() # ---------------------------- def __init__( self , this_ftp_id_id = None , this_server_ip_address = None , this_file_manager_id = None , this_upload = True , this_user_remote_dir = _ftp_posix_separator , this_user_local_dir = "." , this_debug_mode = False , this_start_time = 0 , this_check_size = True ): """ Elementary thread to manage one FTP connection. @param this_ftp_id_id: FTP connection ident @type this_ftp_id_id: FTP object @param this_server_ip_address: remote IP address (for error) @type this_server_ip_address: ascii string @param this_file_manager_id: file manager ident returned by _ftp_open_many_connection() @type this_file_manager_id: gui_scenario_manager object @param this_upload: True = connection for an upload, False = connection for an download @type this_upload: boolean @param this_user_remote_dir: user remote directory @type this_user_remote_dir: ascii string @param this_debug_mode: True = display debug information, False = run silently @type this_debug_mode: boolean @param this_start_time: start time of the FTP transfert (seconds since the epoch) @type this_start_time: float @param this_check_size: True = Check sent and downloaded file size, False = No check @type this_check_size: boolean @return: None """ threading.Thread.__init__(self) self.ftp_id = this_ftp_id_id #: FTP connection to be used self.server_ip_address = this_server_ip_address #: Server IP address self.file_manager_id = this_file_manager_id #: File manager self.debug_mode = this_debug_mode #: Flag for debug purpose self.result = True #: private result/error self.dbg_check_size = this_check_size #: for debug purpose self.start_time = this_start_time #: Keep in mind start time self.last_remote_dir = "" self.ftp_dir_mutex = _ftp_mutex(this_ip_addr = self.server_ip_address) self.user_local_dir = this_user_local_dir #: Current host directory name if (this_user_remote_dir is None) or (this_user_remote_dir == _ftp_posix_separator): self.user_remote_dir = self.ftp_id.pwd() # Use the default user directory else: # Use the POSIX separator for remote directory ! self.user_remote_dir = this_user_remote_dir.replace(os.sep,_ftp_posix_separator) #: User directory on the server self.do_upload = this_upload return # --------------------------- # _ftp_slave_thread.__del__() # --------------------------- def __del__(self): """ For debug purpose @return: None """ self.close_connection() return # ------------------------- # _ftp_slave_thread.error() # ------------------------- def error( self , this_msg ): """ Shortcut access to display an error @param this_msg: input message describing the error @type this_msg: ascii string @return: None """ # Add elapsed running time (for server timer delay purpose) this_msg += "\nRunning time %s"%(gui_common.elapsed_time_ascii( this_start_time = self.start_time)) if self.debug_mode is False: gui_common.save_error( this_msg = this_msg , this_object_class = self , this_file_name = __file__ , this_mail_subject = _ftp_error_tag ) else: gui_common.display_error( this_msg = this_msg , this_object_class = self , this_file_name = __file__ ) self.result = False return # --------------------------- # _ftp_slave_thread.warning() # --------------------------- def warning( self , this_msg ): """ Shortcut access to display a warning @param this_msg: input message describing the error @type this_msg: ascii string @return: None """ # Add elapsed running time (for server timer delay purpose) this_msg += "\nRunning time %s"%(gui_common.elapsed_time_ascii( this_start_time = self.start_time)) if self.debug_mode is False: gui_common.save_warning( this_msg = this_msg , this_object_class = self , this_file_name = __file__ , this_caller_name = gui_common.get_caller_name() ) else: gui_common.display_warning( this_msg = this_msg , this_object_class = self , this_file_name = __file__ , this_caller_name = gui_common.get_caller_name() ) return # ------------------------------------ # _ftp_slave_thread.close_connection() # ------------------------------------ def close_connection( self ): """ Close the FTP connection. @return: None """ _ftp_close_connection( this_ftp_id = self.ftp_id) self.ftp_id = None return # ------------------------------------- # _ftp_slave_thread.change_remote_dir() # ------------------------------------- def change_remote_dir( self , this_new_dir = os.curdir ): """ Change the remote directory. @param this_new_dir: new remote POSIX directory @type this_new_dir: ascii string @return: True = done, False = error detected @rtype: boolean @note: - the directory MUST have been created previously. - the remote directory has the POSIX standard format """ # Force the absolute remote path if this_new_dir == "": # default remote directory this_new_dir = _ftp_posix_separator else: if this_new_dir[0] != _ftp_posix_separator: this_new_dir = _ftp_posix_separator+this_new_dir if self.last_remote_dir == this_new_dir: # Nothing to do ! return True # Be carefull when multiple clients are connected on the same remote host and want to create a new directory... self.ftp_dir_mutex.get(this_ip_addr = self.server_ip_address) err_msg = None try: self.ftp_id.cwd( this_new_dir ) except ftplib.error_temp, e: if "500 can't access directory" in e.args[0]: err_msg = e.args[0] except ftplib.error_reply, e: err_msg = e.args[0] except (ftplib.error_perm,ftplib.error_proto), e: err_msg = e.args[0] except socket.timeout,msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) except socket.error, msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) except gui_common.runing_errors,why: err_msg = str(why) except: err_msg = "ERROR" if err_msg is None: # Yeap ! self.last_remote_dir = this_new_dir else: # Oops ! if ("550 CWD failed" in err_msg) and ("directory not found" in err_msg): # The directory must be created try: self.ftp_id.mkd( dirname = this_new_dir) except (ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e: err_msg = e.args[0] except socket.error, msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) except gui_common.runing_errors,why: err_msg = str(why) except: err_msg = "ERROR" else: err_msg = None if err_msg is not None: err_msg = "mkd('%s') on %s: %s"%( this_new_dir , self.server_ip_address , err_msg ) self.error( this_msg = err_msg) else: # Same player shoots again ! try: self.ftp_id.cwd( this_new_dir ) except ftplib.error_temp, e: if "500 can't access file" in e.args[0]: err_msg = e.args[0] except ftplib.error_reply, e: err_msg = e.args[0] except (ftplib.error_perm,ftplib.error_proto), e: err_msg = e.args[0] except socket.timeout,msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) except socket.error, msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) except gui_common.runing_errors,why: err_msg = str(why) except: err_msg = "ERROR" if err_msg is None: # Yeap ! self.last_remote_dir = this_new_dir else: err_msg = "cwd('%s') on %s: %s"%( this_new_dir , self.server_ip_address , err_msg ) self.error( this_msg = err_msg) else: err_msg = "cwd('%s') on %s: %s"%( this_new_dir , self.server_ip_address , err_msg ) self.error( this_msg = err_msg) # Remote host directory creation should be successful self.ftp_dir_mutex.release(this_ip_addr = self.server_ip_address) return err_msg is None # -------------------------- # _ftp_slave_thread.upload() # -------------------------- def upload( self , this_full_src_file , this_full_dest_file , this_size ): """ FTP upload one file. @param this_full_src_file: full file name on the local host @type this_full_src_file: ascii string @param this_full_dest_file: full file name on the server @type this_full_dest_file: ascii string @param this_size: file size (in bytes) @type this_size: integer @return: True = Done, False = Error detected @rtype: boolean @warning: The ftplib.storbinary() should use socket.MSG_WAITALL when data are sent over the socket. Unfortunatly, this parameter does not exist on Windows. So, sent file may have erroneous size on the destination host... @note: If an error occured during the data transfert, the waiting delay is doubled for each new try. """ if self.debug_mode is True: msg = "%s %s --> %s"%( gui_common.ftp_upload_marker , this_full_src_file , this_full_dest_file ) gui_common.display_screen( this_msg = msg , this_object_class = self , this_file_name = __file__ , this_caller_name = None , this_user_name = None ) # Extract remote parameters (UNIX path format...) path_list = this_full_dest_file.split(_ftp_posix_separator) remote_dir = _ftp_posix_separator.join(path_list[:-1]) remote_file = path_list[-1] # And then, go on to the remote directory (if not already done) if self.change_remote_dir( this_new_dir = remote_dir) is not True: return False # Sometimes,the FTPlib could miss the answer, or the answer could be lost in the network... attempt_number = _ftp_repeat_nb while attempt_number > 0: attempt_number -= 1 err_msg = None # Open the local file try: src_file_id = open( this_full_src_file, 'rb', gui_common.open_buffer_option) except gui_common.runing_errors,why: err_msg = "%s\n%s"%( str(why) , gui_common.which_process_uses_it( this_file = this_full_src_file) ) self.error( this_msg = err_msg) return False except: err_msg = "open(%s,'rb')\n%s"%( this_full_src_file , gui_common.which_process_uses_it( this_file = this_full_src_file) ) self.error( this_msg = err_msg) return False # Upload first try: # Use binary transfert for ASCII and binary files ;-) self.ftp_id.storbinary( cmd = 'STOR %s'%(remote_file) , fp = src_file_id , blocksize = _ftp_block_size ) except socket.timeout,msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) except socket.error,msg: if msg.errno == 0: # <<>> # b'[Errno 0] Error' occures sometime... # so the "226 Successfully transferred "/nemo_tmp/dst_dir/fzs-2015-09-22.log" is ignored !! # and will be treated later whence an error is forseen (incorrect response treatement) ... try: missed_answer = self.ftp_id.getresp() # pop the answer server ! except: # Nothing on the line... err_msg = None else: if missed_answer[0] == '2': err_msg = None else: # It's an error err_msg = missed_answer #print "*** %s *** %s STOR %s --> socket.error '%s'"%(self.name,self.server_ip_address,remote_file,str(msg)) else: # Real error err_msg = gui_common.get_socket_error_msg( this_error = msg) except (ftplib.error_temp, ftplib.error_reply, ftplib.error_perm, ftplib.error_proto),e: # <<>> #--- on server # (000346)30/09/2015 11:27:18 - nemo_pyc (135.238.179.15)> TYPE I # (000346)30/09/2015 11:27:18 - nemo_pyc (135.238.179.15)> 200 Type set to I # (000346)30/09/2015 11:27:48 - nemo_pyc (135.238.179.15)> TYPE I # (000346)30/09/2015 11:27:48 - nemo_pyc (135.238.179.15)> 200 Type set to I # (000346)30/09/2015 11:27:48 - nemo_pyc (135.238.179.15)> PASV # (000346)30/09/2015 11:27:48 - nemo_pyc (135.238.179.15)> 227 Entering Passive Mode (135,120,162,6,202,181) # (000346)30/09/2015 11:27:48 - nemo_pyc (135.238.179.15)> SIZE Handle.exe # (000346)30/09/2015 11:27:48 - nemo_pyc (135.238.179.15)> 213 536256 # (000346)30/09/2015 11:27:48 - nemo_pyc (135.238.179.15)> QUIT # (000346)30/09/2015 11:27:48 - nemo_pyc (135.238.179.15)> 221 Goodbye # (000346)30/09/2015 11:27:48 - nemo_pyc (135.238.179.15)> disconnected. #--- on client: # *** FTP_135.120.162.6_Upload_1 *** 135.120.162.6 STOR Handle.exe --> ftplib.error '550 can't access file.' # *** FTP_135.120.162.6_Upload_1 *** 135.120.162.6 STOR Handle.exe --> ftplib.error '200 Type set to I' # *** FTP_135.120.162.6_Upload_1 *** 135.120.162.6 STOR Handle.exe OK missed_answer = e.args[0] # or e.message for tag_idx in [ "Type set to I" , "Opening data channel" , "SSL connection for data connection established" ]: if tag_idx in missed_answer: # Ignore the current message and take the next pending one (if any) try: missed_answer = self.ftp_id.getresp() # pop the answer server ! except: # Nothing on the line... missed_answer = '2xx' break # Next tag_idx continue if missed_answer[0] == '2': err_msg = None #print "*** %s *** %s STOR %s --> ftplib.error '%s'"%(self.name,self.server_ip_address,remote_file,str(e)) else: # It's an error err_msg = missed_answer except gui_common.runing_errors,why: err_msg = str(why) except: err_msg = "storbinary()" else: err_msg = None # Done with this file try: src_file_id.close() except: pass if _ftp_repeat_command( this_err_msg = err_msg) is False: break # while attempt_number time.sleep(_ftp_repeat_waiting_delay) continue if err_msg is not None: # Oops ! full_err_msg = "%s %s: STOR '%s' --> '%s': %s"%( gui_common.ftp_upload_marker , self.server_ip_address , this_full_src_file , remote_file , err_msg ) self.error( this_msg = full_err_msg) else: # The FTP upload has been done if self.dbg_check_size is True: # Get the remote file size remote_file_size = 0 # Because the Windows OS may take a while to update its internal buffers, we must be carrefull... attempt_number = _ftp_repeat_nb while attempt_number > 0: attempt_number -= 1 try: remote_file_size = self.ftp_id.size(remote_file) except (socket.timeout,socket.error),msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) if err_msg == "": err_msg = None except (ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto),e: err_msg = e.args[0] if err_msg == "": err_msg = None except gui_common.runing_errors,why: err_msg = str(why) except: err_msg = "ftp.size(%s)"%(remote_file) else: # We have read a remote size... if this_size != remote_file_size: # Different file sizes... err_msg = "%s: local '%s' = %d, remote '%s' = %d"%( _file_size_tag , this_full_src_file, this_size , remote_file, remote_file_size ) else: # Yeap ! err_msg = None break if _ftp_repeat_command( this_err_msg = err_msg) is False: break # while attempt_number time.sleep(_ftp_repeat_waiting_delay) continue # The result is... if err_msg is not None: # Oops ! full_err_msg = "%s %s: SIZE '%s': %s"%( gui_common.ftp_upload_marker , self.server_ip_address , remote_file , err_msg ) self.error( this_msg = full_err_msg) return err_msg is None # ---------------------------- # _ftp_slave_thread.download() # ---------------------------- def download( self , this_full_src_file , this_full_dest_file , this_size ): """ FTP download one file. @param this_full_src_file: full file name on the server @type this_full_src_file: ascii string @param this_full_dest_file: full file name on the local host @type this_full_dest_file: ascii string @param this_size: file size (in bytes) @type this_size: integer @return: True = Done, False = Error detected @rtype: boolean @note: If an error occured during the data transfert, the waiting delay is doubled for each new try. """ if self.debug_mode is True: msg = "%s %s --> %s"%( gui_common.ftp_download_marker , this_full_src_file , this_full_dest_file ) gui_common.display_screen( this_msg = msg , this_object_class = self , this_file_name = __file__ , this_caller_name = None , this_user_name = None ) if this_size == _ftp_dir_size: # Local directory has been already created return True if this_size == 0: # Ignore empty remote file for downloading return True # Extract remote parameters (UNIX path format...) path_list = this_full_src_file.split(_ftp_posix_separator) remote_dir = _ftp_posix_separator.join(path_list[:-1]) remote_file = path_list[-1] if remote_dir != "": # And then, go on to the remote directory (if not already done) if self.change_remote_dir( this_new_dir = remote_dir) is not True: return False # Get remote file err_msg = None dst_file_id = None dst_dir = os.path.dirname(this_full_dest_file) if os.path.isdir(dst_dir) is False: # Create the local directory if gui_common.build_directories( this_directory_list = [dst_dir,]) is False: err_msg = "Can't create '%s' directory"%(dst_dir) self.error( this_msg = err_msg) return False # Sometimes,the FTPlib could miss the answer, or the answer could be lost in the network... attempt_number = 0 while attempt_number < _ftp_repeat_nb: attempt_number += 1 err_msg = None # Open the destination file to store its content try: dst_file_id = open( this_full_dest_file, 'wb', gui_common.open_buffer_option) except gui_common.runing_errors,why: err_msg = "%s\n%s"%( str(why) , gui_common.which_process_uses_it( this_file = this_full_dest_file) ) if "[Errno 2]" in err_msg: # Should never occur err_msg += gui_common.test_path( this_path = this_full_dest_file , this_directory_only = True ) except: err_msg = "open(%s, 'wb')\n%s"%( this_full_dest_file , gui_common.which_process_uses_it( this_file = this_full_dest_file) ) if err_msg is not None: # Oops ! Can't open a file for writting on my host... full_err_msg = "%s from %s: %s"%( gui_common.ftp_download_marker , self.server_ip_address , err_msg ) self.error( this_msg = full_err_msg) break # Retreive back the remote file content timeout_occures = False try: # Use binary transfert for ASCII and binary files ;-) self.ftp_id.retrbinary( cmd = 'RETR %s'%(remote_file) , callback = dst_file_id.write , blocksize = _ftp_block_size ) except socket.timeout,msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) timeout_occures = True except socket.error, msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) except (ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e: # The error "550 can't access file" can occure when we try to download a file which is already opened for writing by an another process err_msg = e.args[0] if "550" in err_msg: err_msg += ". The file is maybe opened for writing by an another process..." except gui_common.runing_errors,why: err_msg = str(why) except: err_msg = "ftp.retrbinary()" else: err_msg = None # Close the file before threating the error ;-) gui_common.force_file_synchro( this_file_id = dst_file_id) try : dst_file_id.close() except: pass # The result is... if timeout_occures is not True: if err_msg is None: break if _ftp_repeat_command( this_err_msg = err_msg) is False: break # while attempt_number time.sleep(_ftp_repeat_waiting_delay) continue if err_msg is not None: # Oops ! full_err_msg = "%s %s: '%s' --> RETR '%s': %s"%( gui_common.ftp_download_marker , self.server_ip_address , this_full_src_file , this_full_dest_file , err_msg ) self.error( this_msg = full_err_msg) else: # Download done, no error ! if self.dbg_check_size is True: # Get host file size try: file_host_size = os.path.getsize(this_full_dest_file) except gui_common.runing_errors,why: # Should never occure err_msg = str(why) except: # Idem ! err_msg = "os.path.getsize(%s)"%(this_full_dest_file) else: if file_host_size != this_size: err_msg = "%s: host %s = %d, remote %s = %d"%( _file_size_tag , this_full_dest_file, file_host_size , this_full_src_file, this_size ) if err_msg is not None: full_err_msg = "%s %s: SIZE: %s"%( gui_common.ftp_download_marker , self.server_ip_address , err_msg ) self.error( this_msg = full_err_msg) return err_msg is None # ----------------------- # _ftp_slave_thread.run() # ----------------------- def run(self): """ Main body of the thread dedicated to a FTP elementary transfert. """ file_to_transfert_info = self.file_manager_id.get_waiting() while file_to_transfert_info is not None: # I have a file to transfert (full_src_file, full_dest_file, file_size) = file_to_transfert_info # Go, go, go ! if self.do_upload is True: # Upload wanted tranfert_result = self.upload( this_full_src_file = full_src_file , this_full_dest_file = full_dest_file , this_size = file_size ) else: # Download wanted tranfert_result = self.download( this_full_src_file = full_src_file , this_full_dest_file = full_dest_file , this_size = file_size ) if tranfert_result is False: # Can't transfert. Mark it as 'running' again. It's craps... I know :-( self.file_manager_id.add_running( this_scenario_name = file_to_transfert_info) # Pray for a succes by an another thread break else: # Forever loop until all files have been transfered file_to_transfert_info = self.file_manager_id.get_waiting() # while file_to_transfert_info continue # Close the FTP connection self.close_connection() return # ---------------------------------------------- # Private Function : _ftp_open_many_connection() # ---------------------------------------------- def _ftp_open_many_connection( this_address , this_login , this_password , this_timeout = gui_common.ftp_timeout_s , this_upload = True , this_user_remote_dir = _ftp_posix_separator , this_max_connection = 2 , this_debug_mode = False , this_nickname = None , this_check_size = True , this_use_sftp = False ): """ Open at most some connections on a dedicated serveur and associated a thread to manage each. @param this_address: remote IP address @type this_address: ascii string @param this_login: user login @type this_login: ascii string @param this_password: user password @type this_password: ascii string @param this_timeout: maximal timeout in seconds for blocking operations like the connection attempt @type this_timeout: int @param this_upload: True = connection for an upload, False = connection for an download @type this_upload: boolean @param this_user_remote_dir: remote user directory @type this_user_remote_dir: ascii string @param this_max_connection: number of connections to speed up the transfert @type this_max_connection: integer @param this_debug_mode: True = display debug information, False = run silently @type this_debug_mode: boolean @param this_nickname: connection nickname @type this_nickname: ascii string @param this_check_size: True = Check sent and downloaded file size, False = No check @type this_check_size: boolean @param this_use_sftp: True = use FTPS, False = use normal FTP @type this_use_sftp: boolean @return: (file manager ident, ftp thread ident) tuple @rtype: list of tuple @note: the number of connection may be less than the one expected on detected remote error. """ answer = [] file_manager_id = gui_scenario_manager.scenario_manager( this_debug_mode = this_debug_mode) # Be sure to have an integer this_timeout = int(this_timeout) # Increase timeout if neccessary if gui_common.vpn_ip_addr() is True: # When running with VPN this_timeout *= 2 start_time = time.time() cnx_idx = 0 if this_upload is True: ud = "Upload" else: ud = "Download" while this_max_connection > 0: this_max_connection -= 1 err_msg = None attempt_number = _ftp_repeat_nb # Be carefull: the server may be temporarily full while attempt_number > 0: attempt_number -= 1 # Open a connection on the server try: if this_use_sftp is True: # FTPS ftp_id = ftplib.FTP_TLS( host = this_address , user = this_login , passwd = this_password , keyfile = None , certfile = _ftps_certificate_file_name , timeout = this_timeout ) else: # Default FTP ftp_id = ftplib.FTP( host = this_address , user = this_login , passwd = this_password , timeout = this_timeout ) except (ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e: err_msg = e.args[0] except socket.error, msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) except gui_common.runing_errors,why: err_msg = str(why) except: err_msg = "ftplib.FTP()" else: # Yes ! err_msg = None break if _ftp_repeat_command( this_err_msg = err_msg) is False: break # while attempt_number time.sleep(_ftp_repeat_waiting_delay) continue if err_msg is not None: # Oops ! No FTP connection... msg = "Unable to open FTP connection (addr: %s, login: %s, password: %s, timeout: %ds)\n| %s"%( this_address , this_login , this_password , this_timeout , err_msg ) gui_common.save_error( this_msg = msg , this_object_class = None , this_file_name = __file__ , this_mail_subject = "%s %s: %s"%(_ftp_error_tag,this_address,err_msg) ) # We can not open as many connection as we would like... Run whith the openned ones as 'fail-soft' mode ;-) return answer else: if this_use_sftp is True: # Switch ON secure data connection ftp_id.prot_p() # Set FTP socket options if gui_common.set_socket_options( this_socket_id = ftp_id.sock , this_timeout_s = this_timeout , this_device = gui_common.io_device_ethernet0 , this_server_purpose = False , this_debug = this_debug_mode , this_mtu = gui_common.ip_mtu ) is False: _ftp_close_connection( this_ftp_id = ftp_id) return answer # Set FTP debug trace level # 0: no debugging output (default) # 1: print commands and responses but not body text etc. # 2: also print raw lines read and sent before stripping CR/LF if this_debug_mode is True: ftp_id.set_debuglevel( level = 1 ) else: ftp_id.set_debuglevel( level = 0 ) # Force 'passive mode' ftp_id.set_pasv( True ) # Create a dedicated thread which manage the transfert. It will be launched later by the caller. ftp_thread_id = _ftp_slave_thread( this_ftp_id_id = ftp_id , this_server_ip_address = this_address , this_file_manager_id = file_manager_id , this_upload = this_upload , this_user_remote_dir = this_user_remote_dir , this_start_time = start_time , this_debug_mode = this_debug_mode , this_check_size = this_check_size ) cnx_idx += 1 if this_nickname is None: thread_name = "FTP_%s_%s_%d"%( this_address , ud , cnx_idx ) else: thread_name = "FTP_%s_%s_%s_%d"%( this_address , this_nickname , ud , cnx_idx ) ftp_thread_id.setName(thread_name) info = (file_manager_id, ftp_id, ftp_thread_id) answer.append(info) # While this_max_connection continue return answer # ----------------------------------------------- # Private Function : _ftp_close_connection_list() # ----------------------------------------------- def _ftp_close_connection_list( this_list ): """ Close opened FTP connection thanks to _ftp_open_many_connection() @param this_list: (file manager ident, ftp thread ident) tuple @type this_list: list of tuples comming from _ftp_open_many_connection() """ for ftp_cnx_info in this_list : (file_manager_id, ftp_id, ftp_thread_id) = ftp_cnx_info _ftp_close_connection( this_ftp_id = ftp_id) # Next ftp_cnx_info continue return # -------------------------------------- # Function : _ftp_get_directory_detail() # -------------------------------------- def _ftp_get_directory_detail( this_ftp_id = None , this_dir = None ): """ Get detailed remote directory info (Filezilla Server) @param this_ftp_id: open FTP connection @type this_ftp_id: ftplib object @param this_ftp_id: remote directory. if None, the current one is used @type this_ftp_id: ascii string @return """ # Go to the given directory if this_dir is not None: this_ftp_id.cwd( this_dir ) # Ask information file_list = [] this_ftp_id.dir(file_list.append) # The Filezilla Server answer format looks like: # -rw-r--r-- 1 ftp ftp 1608780 Aug 29 2012 FileZilla Server.log # -rw-r--r-- 1 ftp ftp 2634337 Mar 10 17:17 fzs-2015-03-10.log # -rw-r--r-- 1 ftp ftp 7296369 Mar 11 17:39 fzs-2015-03-11.log current_year = int(time.strftime("%Y", time.gmtime())) answer = [] for info_idx in file_list: splitted_line = info_idx.split() user_group_other_rights = splitted_line[0] directory_flag = user_group_other_rights[0] # 'd' or '-' user_group_other_rights = user_group_other_rights[1:] inode_number = splitted_line[1] user_name = splitted_line[2] group_name = splitted_line[3] size = int(splitted_line[4]) month = splitted_line[5] date = int(splitted_line[6]) if ':' in splitted_line[7]: # Hour only hour_minute = splitted_line[7] year = current_year else: # Year only hour_minute = "00:00" year = int(splitted_line[7]) file = " ".join(splitted_line[8:]) element = (directory_flag, size, year, month, date, hour_minute, file) answer.append(element) # Next info_idx continue return answer # --------------------------------- # Function : _ftp_get_server_info() # --------------------------------- def _ftp_get_server_info( this_ftp_id ): """ Return the FTP server information like name and features @param this_ftp_id: open FTP connection @type this_ftp_id: ftplib object @return: remote FTP server name and features @rtype: ascii string """ err_msg = None try: system_info = this_ftp_id.sendcmd( cmd = 'SYST') except (ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e: err_msg = e.args[0] except socket.error, msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) else: expected_answer = "215 " if system_info.startswith(expected_answer) is True: # Yeap ! We have the offical system name like '215 UNIX emulated by FileZilla', ignore the number... system_info = system_info[len(expected_answer):] if err_msg is not None: gui_common.save_error( this_msg = "sendcmd('SYST') : %s"%(err_msg) , this_object_class = this_ftp_id , this_file_name = __file__ , this_mail_subject = "%s: %s"%(_ftp_error_tag,err_msg) ) system_info = "Unknown system" # Feature Negotiation... [RFC2389] feature_list = [] try: response = this_ftp_id.sendcmd('FEAT') except: # Command not supported ! Snif pass else: # The Filezilla server answer looks like: b'211-Features:\n MDTM\n REST STREAM\n SIZE\n MLST type*;size*;modify*;\n MLSD\n UTF8\n CLNT\n MFMT\n211 End' expected_answer = "211" if response.startswith(expected_answer) is True: # Yeap ! ignore the number and the 'End'... feature_list = [idx for idx in response.split(':')[1].split('\n')[:-1] if idx != ""] # --------------------------------------------- # <<>> # Synopsis # The OPTS command is a required command if the FEAT command is also implemented. # The OPTS command is used to provide additional information for extended features supported by the FTP server. # Description # The OPTS command will be followed by the name of the command requiring additional information. # Following the command being configured will be additional parameters that have meaning only in the context of the command being configured. # FTP Voyager currently issues the OPTS command under two circumstances: # MODE - FTP Voyager will issue an OPTS MODE command to servers to configure extended options for a given transfer mode. Most commonly MODE is implemented in the "MODE Z LEVEL X" command for supporting on-the-fly compression to configure the level of compression to use for the session where 'X' is a number between 1 (fastest, least compression) and 10 (slowest, most compression). FTP Voyager uses a default value of 6. # MLST - FTP Voyager may issue an OPTS MLST command to servers supporting the MLST command in order to configure the amount of information sent by the server in response to the MLST command. It is issued in the format OPTS MLST Info1;Info2;Info3 etc, with each type of information separated by a semicolon. Information types can include Type (filetype), Size (filesize), Modify (last modification date), Create (date of file creation), and more. The information types the server supports are sent to the client as part of the server's response to the FEAT command, and used to determine the contents of the OPTS MLST command issued to the server. # UTF8 - Configures the server to enable (ON) or disable (OFF) UTF-8 encoding which is useful for non-ANSI character sets. This is very useful for Asian file names and paths. # # COMMAND:> OPTS MLST Type;Size;Modify;UNIX.mode;UNIX.owner;UNIX.group; # 200 MLST OPTS Type;Size;Modify;UNIX.mode;UNIX.owner;UNIX.group; # --------------------------------------------- # <<>> # Synopsis # The MFMT command is used to modify a file or folder's last modified date and time information. MFMT duplicates similar functionality implemented by the MDTM command. # Description # Traditionally, when a file or folder is uploaded to an FTP server, the last modified date and time of the file or folder is set to the transfer date and time. # Using MFMT, FTP clients can inform supporting FTP servers of the proper last modified date and time to use for the file or folder. # The format of the command is MFMT YYYYMMDDHHMMSS path, where: # YYYY - the 4-digit year # MM - the 2-digit month # DD - the 2-digit day of the month # HH - the hour in 24-hour format # MM - the minute # SS - the seconds # All times are represented in GMT/UTC. # --------------------------------------------- # <<>> # Synopsis # The MDTM command implemented by FTP is used to preserve a file's original date and time information after file transfer. # Description # Traditionally, when a file is uploaded to an FTP server, the date/time of the file is set to the transfer date/time. # Using MDTM, FTP Voyager can inform supporting FTP servers of the proper date/time to use for the file. # The format of the command is MDTM YYYYMMDDHHMMSS, where: # YYYY - the 4-digit year # MM - the 2-digit month # DD - the 2-digit day of the month # HH - the hour in 24-hour format # MM - the minute # SS - the seconds # --------------------------------------------- answer = (system_info, feature_list) return answer # ----------------------------------------- # Private Function : _ftp_get_remote_info() # ----------------------------------------- def _ftp_get_remote_info( this_ftp_id = None , this_ftp_tag = None , this_tag_selector = [] , this_remote_directory_flag = None ): """ Return the listing of a remote file / directory @param this_ftp_id: open FTP connection @type this_ftp_id: ftplib object @param this_ftp_tag: target directory or file to list @type this_ftp_tag: ascii string @param this_tag_selector: optional tag selector @type this_tag_selector: list of ascii string @param this_remote_directory_flag: False = remote file, True = remote directory, None = to be determined @type this_remote_directory_flag: boolean @return: list of (full remote path name, is_directory, file_size) @rtype: list of tuple (ascii string, boolean, integer) @note: - use the standard MLSD command instead of LIST (result os dependant) - NLST returns the file names without any type :-( - The MLST command only provides detailed information about a single file or folder while the MLSD can provide information about multiple files and folders. """ callback_answer_list = [] # --------------------------------- # Private: ftp_mlsd_mlst_callback() # --------------------------------- def ftp_mlsd_mlst_callback( this_input ): """ Analyse the standard MLSD & MLST answer (local function for recursivity) Update 'answer' in relation with the incoming data The FTP MLSD answer should looks like: - type=dir;modify=20100114090849; Lib - type=file;modify=20100114090849;size=1190652; swig.exe the MLST answer should looks like: - type=file;size=386456;modify=20150317130322; /AGERMANE.3.zip """ # Be carefull: if the command has been sent with retbinary we have all result in one answer with '\n\r' as separator ! for input_idx in this_input.split(gui_common.char_crlf): if input_idx == "": # Ignore it ! continue # For each record element full_splitted = [element_idx.strip() for element_idx in input_idx.split(';')] # Get name and apply a filter name = full_splitted[-1].strip() if name[0] == _ftp_posix_separator: name = name[1:] match = True for tag_idx in this_tag_selector: if tag_idx not in name: # Discard it ! match = False break # Next tag_idx continue if match is True: # Get 'type' info: file or directory splitted_record = full_splitted[0].split('=') if splitted_record[0] != "type": gui_common.save_error( this_msg = "unexpected format: %s"%(this_input) , this_object_class = this_ftp_id , this_file_name = __file__ , this_mail_subject = _ftp_error_tag ) return is_a_directory = (splitted_record[1] == "dir") # Get file size (not available for directory) file_size = 0 if is_a_directory is False: # !!! 'size' & 'modify' parameters are inverted between MLST & MLSD results !!! for idx in range(1,len(full_splitted)-1): splitted_record = full_splitted[idx].split('=') if splitted_record[0] == "size": # Yeap ! file_size = int(splitted_record[1]) break # Next idx continue # The result is... result = (name, is_a_directory, file_size) callback_answer_list.append(result) # Next input_idx continue return # ------------------------------- # Private: sort_callback_result() # ------------------------------- def sort_callback_result(tag1,tag2): (name1, is_a_directory1, file_size1) = tag1 (name2, is_a_directory2, file_size2) = tag2 if is_a_directory1 is True: if is_a_directory2 is True: # 2 directories --> sort by directory name if name1 < name2: return -1 else: return 1 else: # Dir 1, file 2 return 1 else: if is_a_directory2 is True: # File 1, dir 2 return -1 else: # 2 files --> sort by by file name if name1 < name2: return -1 else: return 1 return 0 # # Body: _ftp_get_remote_info() # answer = [] err_msg = None # Do some checks if this_ftp_tag is None: # All default directory content this_ftp_tag = _ftp_posix_separator remote_directory = _ftp_posix_separator # FTP or SFTP ? try: sftp_in_use = this_ftp_id._prot_p except: sftp_in_use = False # What is the remote target ? if this_remote_directory_flag is None: # Verify if it's a remote file or a remote directory try: remote_file_size = this_ftp_id.size( this_ftp_tag) except ftplib.error_perm, e: # It's a directory this_remote_directory_flag = True else: # It's a file. Ask remote file information this_remote_directory_flag = False # Get remote information if this_remote_directory_flag is True: remote_directory = this_ftp_tag+_ftp_posix_separator # Go to the selected directory try: this_ftp_id.cwd( dirname = this_ftp_tag) except (ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e: err_msg = e.args[0] except socket.error, msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) else: # Ask remote directory information try: if sftp_in_use is True: result = this_ftp_id.retrbinary( cmd = 'MLSD' , callback = ftp_mlsd_mlst_callback ) else: result = this_ftp_id.retrlines( cmd = 'MLSD' , callback = ftp_mlsd_mlst_callback ) except (ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e: err_msg = e.args[0] except socket.error, msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) if err_msg is not None: gui_common.save_error( this_msg = "retrlines('MLSD'): %s"%(err_msg) , this_object_class = this_ftp_id , this_file_name = __file__ , this_mail_subject = "%s: %s"%(_ftp_error_tag,err_msg) ) return answer else: remote_directory = "" # Obtain remote file information try: result = this_ftp_id.sendcmd( cmd = 'MLST '+this_ftp_tag) except (ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e: err_msg = e.args[0] except socket.error, msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) else: # The MLST answer looks like: b'250-Listing /AGERMANE.3.zip\n type=file;size=386456;modify=20150317130322; /AGERMANE.3.zip\n250 End' splitted_answer = result.split('\n') if '250' in splitted_answer[0]: # Yeap ! this_remote_directory_flag = False ftp_mlsd_mlst_callback ( this_input = splitted_answer[1]) else: err_msg = "Unexpected answer: '%s'"%(result) if err_msg is not None: gui_common.save_error( this_msg = "sendcmd('MLST'): %s"%(err_msg) , this_object_class = ftp_id , this_file_name = __file__ , this_mail_subject = "%s: %s"%(_ftp_error_tag,err_msg) ) return answer # Analyse the result of the file or of the current directory... for remote_info_idx in callback_answer_list: try: (name, is_directory, file_size) = remote_info_idx except gui_common.runing_errors,why: err_msg = "Unexpected 'local_info_idx' format: %s - '%s'"%(str(why), str(remote_info_idx)) gui_common.save_error( this_msg = err_msg , this_object_class = this_ftp_id , this_file_name = __file__ , this_mail_subject = "%s: %s"%(_ftp_error_tag,err_msg) ) return answer if is_directory is True: # It's a directory, go on recursively if this_ftp_tag is None: new_dir = _ftp_posix_separator+name else: new_dir = name answer2 = _ftp_get_remote_info( this_ftp_id = this_ftp_id , this_ftp_tag = new_dir , this_tag_selector = this_tag_selector , this_remote_directory_flag = is_directory ) # Build a full path answer for remote_info_jdx in answer2: try: (name2, is_directory2, file_size2) = remote_info_jdx except gui_common.runing_errors,why: err_msg = "Unexpected 'remote_info_jdx' format: %s - '%s'"%(str(why), str(remote_info_jdx)) gui_common.save_error( this_msg = err_msg , this_object_class = this_ftp_id , this_file_name = __file__ , this_mail_subject = "%s: %s"%(_ftp_error_tag,err_msg) ) return answer remote_info_jdx = (this_ftp_tag+_ftp_posix_separator+name2, is_directory2, file_size2) answer.append(remote_info_jdx) # Next remote_info_jdx continue # Return to the parent directory try: result = this_ftp_id.sendcmd( cmd = 'CDUP') except (ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e: err_msg = e.args[0] except socket.error, msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) if err_msg is not None: gui_common.save_error( this_msg = "sendcmd(CDUP): %s"%(err_msg) , this_object_class = this_ftp_id , this_file_name = __file__ , this_mail_subject = "%s: %s"%(_ftp_error_tag,err_msg) ) return answer # Do a difference between directory size and empty file size file_size = _ftp_dir_size # Answer with full path name remote_info_idx = (remote_directory+name, is_directory, file_size) answer.append(remote_info_idx) # Next remote_info_idx continue return answer # --------------------------- # Private Class : _ftp_thread # --------------------------- class _ftp_thread( threading.Thread): """ Independant FPT thread to upload or download several host in parallele @sort: _*,a*,b*,c*,d*,e*,f*,g*,h*,i*,j*,k*,l*,m*,n*,o*,p*,q*,r*,s*,t*,u*,v*,w*,x*,y*,z* """ # ---------------------- # _ftp_thread.__init__() # ---------------------- def __init__( self , this_address = "1.2.3.4" , this_login = "dummy_login" , this_password = "dumthr_password" , this_timeout = gui_common.ftp_timeout_s , this_src_path = None , this_dst_path = None , this_direction= _upload_cmd , this_cleanup_flag = False , this_debug_mode = False ): """ FTP thread initialisation @param this_address: address of the server @type this_address: ascii @param this_login: remote user login @type this_login: ascii @param this_password: remote user password @type this_password: ascii @param this_timeout: timeout in seconds for blocking operations like the connection attempt @type this_timeout: real @param this_src_path: source path name (file or directory). A wildcard '*' is allowed. @type this_src_path: ascii string @param this_dst_path: server destination path name @type this_dst_path: ascii string @param this_direction: 'Upload' or 'Download' transfert direction @type this_direction: ascii string @param this_cleanup_flag: force to clean up the destination directory (True) or not (False) @type this_cleanup_flag: boolean @param this_debug_mode: True = with debug mode, False = silent @type this_debug_mode: boolean @return: None """ # Initialise my thread self.thread_id = threading.Thread.__init__(self) self.result = True self.cleanup_flag = this_cleanup_flag # Save input parameters self.address = this_address self.login = this_login self.timeout = this_timeout self.src_path = this_src_path self.dst_path = this_dst_path self.direction = this_direction self.debug_mode = this_debug_mode # Force the Lab server password if possible if this_login == gui_common.main_login: self.password = gui_common.get_root_password( this_ip_address = self.address) if self.password is None: # It's an another host, use the given password self.password = this_password else: # Use the given password self.password = this_password return # ------------------- # _ftp_thread.error() # ------------------- def error( self , this_msg ): """ Shortcut access to display an error @param this_msg: input message describing the error @type this_msg: ascii string @return: None """ gui_common.save_error( this_msg = this_msg , this_object_class = self , this_file_name = __file__ , this_mail_subject = _ftp_error_tag ) self.result = False return # ------------------------ # _ftp_thread.save_trace() # ------------------------ def save_trace( self , this_msg , this_caller_name ): """ Shortcut access to display an error @param this_msg: input message describing the error @type this_msg: ascii string @param this_caller_name: caller function name @type this_caller_name: ascii string @return: None """ gui_common.save_trace( this_msg = this_msg , this_object_class = self , this_file_name = __file__ , this_caller_name = this_caller_name ) return # ----------------- # _ftp_thread.run() # ----------------- def run(self): """ FTP thread body @return: None """ if self.direction == _upload_cmd: # Upload wanted if self.cleanup_flag is True: # Clean Up wanted on the remote host if self.debug_mode is True: self.save_trace( this_msg = "Try to remove server directory %s : %s"%(self.address,self.dst_path) , this_caller_name = "run()" ) self.result = ftp_remove( this_address = self.address , this_login = self.login , this_password = self.password , this_timeout = self.timeout , this_dst_path = self.dst_path ) if self.result is not True: self.error( this_msg = "Remove server directory %s : %s"%(self.address,self.dst_path)) else: if self.debug_mode is True: self.save_trace( this_msg = "Remove server directory %s done : %s"%(self.address,self.dst_path) , this_caller_name = "run()" ) # And then upload ! self.result = ftp_upload( this_address = self.address , this_login = self.login , this_password = self.password , this_timeout = self.timeout , this_src_path = self.src_path , this_dst_path = self.dst_path ) if self.result is True: if self.debug_mode is True: self.save_trace( this_msg = "Upload server %s done"%(self.address) , this_caller_name = "run()" ) return if self.direction == _download_cmd: if self.cleanup_flag is True: # Clean Up wanted on the running host if os.path.isdir(self.dst_path) is True: # And the directory already exists self.result = gui_common.remove_directory( this_target = self.dst_path+os.sep+"*") if self.result is not True: # Oops ! return # Retreive back all files and directories self.result = ftp_download( this_address = self.address , this_login = self.login , this_password = self.password , this_timeout = self.timeout , this_src_path = self.src_path , this_dst_path = self.dst_path ) if self.result is not True: self.error( this_msg = "Download server %s with %s"%(self.address,self.src_path)) else: if self.debug_mode is True: self.save_trace( this_msg = "Download from server %s done"%(self.address) , this_caller_name = "run()" ) return # Should never occure self.error( this_msg = "Unknown command: %s"%(self.direction)) return ### ---------------- ### ### Public functions ### ### ---------------- ### # ----------------------- # Function : ftp_remove() # ----------------------- def ftp_remove( this_address = "1.2.3.4" , this_login = "dummy_login" , this_password = "dummy_password" , this_timeout = gui_common.ftp_timeout_s , this_dst_path = None , this_debug_mode = False , this_use_sftp = False ): """ FTP Remove recursivly file(s) and directory(ies). Use recursivity when a directory is found. @param this_address: address of the server @type this_address: ascii @param this_login: remote user login @type this_login: ascii @param this_password: remote user password @type this_password: ascii @param this_timeout: timeout in seconds for blocking operations like the connection attempt @type this_timeout: real @param this_dst_path: local destination directory @type this_dst_path: ascii string @param this_debug_mode: current debug mode @param this_use_sftp: True = use FTPS, False = use normal FTP @type this_use_sftp: boolean @type this_debug_mode: boolean @return: True : transfert done / False : error @rtype: boolean """ #------------------------------- # local function for recursivity #------------------------------- def recursive_ftp_remove( this_dst_path , this_is_a_directory = False ): """ local function for recursivity calls @param this_src_path: server source directory @type this_src_path: ascii string @param this_is_a_directory: private parameter: True = directory, False = file (could not exist !) @type this_is_a_directory: boolean """ answer = True # Be careful with the first call... ;-)* dst_dir_list = [] err_msg = None if this_is_a_directory is False: # Determine if it's a file or a directory try: # Is 'file' file or directory ? remote_file_size = ftp_id.size(this_dst_path) # If here : it's a file and it exists dst_dir_list.append(this_dst_path) except ftplib.error_perm, e: # It's a directory or the file does not exist ! recursive_ftp_remove( this_dst_path = this_dst_path , this_is_a_directory = True ) except (ftplib.error_temp,ftplib.error_reply,ftplib.error_proto), e: err_msg = e.args[0] except socket.error, msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) except gui_common.runing_errors,why: err_msg = str(why) if err_msg is not None: gui_common.save_error( this_msg = "delete(%s) on %s : %s"%(this_dst_path,this_address,err_msg) , this_object_class = ftp_id , this_file_name = __file__ , this_mail_subject = "%s: %s"%(_ftp_error_tag,err_msg) ) return False else: # It's a directory or the file does not exist ! try: dst_dir_list = ftp_id.nlst(this_dst_path) except (ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e: # Be careful with '550 Directory not found' err_msg = str(e.args[0]) if '550' in err_msg: # The file does not exist ! It wasn't a directory... return True except socket.error, msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) except gui_common.runing_errors,why: err_msg = str(why) if err_msg is not None: gui_common.save_error( this_msg = "nlst(%s) on %s : %s"%(this_dst_path,this_address,err_msg) , this_object_class = ftp_id , this_file_name = __file__ , this_mail_subject = "%s: %s"%(_ftp_error_tag,err_msg) ) return False # Exam the result list for dst_file in dst_dir_list: # Display debug information (if wanted) if this_debug_mode is True: gui_common.save_trace( this_msg = "recursive_ftp_remove(%s) : %s"%(this_address,dst_file) , this_object_class = None , this_file_name = None , this_caller_name = "recursive_ftp_remove()" ) # Is it a file or a directory ? try: remote_file_size = ftp_id.size(dst_file) # If here : it's a file. Remove it. attempt_number = _ftp_repeat_nb while attempt_number > 0: attempt_number -= 1 try: ftp_id.delete(dst_file) except (ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e: err_msg = e.args[0] except socket.error, msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) except gui_common.runing_errors,why: err_msg = str(why) else: err_msg = None break if _ftp_repeat_command( this_err_msg = err_msg) is False: break # while attempt_number time.sleep(_ftp_repeat_waiting_delay) continue if err_msg is not None: gui_common.save_error( this_msg = "recursive_ftp_remove(%s) -> delete(%s) %d bytes: %s"%(this_address,this_dst_path,remote_file_size,err_msg) , this_object_class = ftp_id , this_file_name = __file__ , this_mail_subject = "%s: %s"%(_ftp_error_tag,err_msg) ) return False except ftplib.error_perm, e: # 5xx errors, It's may be a directory or it does not exist... if recursive_ftp_remove( dst_file , this_is_a_directory = True ) is not True: return False # Remove the directory attempt_number = _ftp_repeat_nb while attempt_number > 0: attempt_number -= 1 try: ftp_id.rmd(dst_file) except ftplib.all_errors, e: err_msg = e.args[0] except socket.error, msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) except gui_common.runing_errors,why: err_msg = str(why) else: err_msg = None break # Do a last control... in case of '550 Directory not found' if "Directory not found" in err_msg: # The error is normal in this case ! It should occure during the first call ;-) err_msg = None break if _ftp_repeat_command( this_err_msg = err_msg) is False: break # while attempt_number time.sleep(_ftp_repeat_waiting_delay) continue if err_msg is not None: gui_common.save_error( this_msg = "recursive_ftp_remove(%s) -> rmd(%s) : %s"%(this_address,this_dst_path,err_msg) , this_object_class = ftp_id , this_file_name = __file__ , this_mail_subject = "%s: %s"%(_ftp_error_tag,err_msg) ) return False except (ftplib.error_temp,ftplib.error_reply,ftplib.error_proto), e: err_msg = e.args[0] except socket.error, msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) except gui_common.runing_errors,why: err_msg = str(why) except : err_msg = "???" if err_msg is not None: gui_common.save_error( this_msg = "recursive_ftp_remove(%s) -> delete(%s) : %s"%(this_address,this_dst_path,err_msg) , this_object_class = ftp_id , this_file_name = __file__ , this_mail_subject = "%s: %s"%(_ftp_error_tag,err_msg) ) return False # Next dst_file continue return True # # Main body of ftp_remove() # if this_debug_mode is True: gui_common.save_trace( this_msg = "ftp_remove(%s) destination path %s, login: %s, pass: %s"%(this_address,this_dst_path,this_login,this_password) , this_object_class = None , this_file_name = __file__ , this_caller_name = "ftp_remove()" ) err_msg = None # Open the FTP connection try: if this_use_sftp is True: # FTPS ftp_id = ftplib.FTP_TLS( host = this_address , user = this_login , passwd = this_password , keyfile = None , certfile = _ftps_certificate_file_name , timeout = this_timeout ) else: # Default FTP ftp_id = ftplib.FTP( host = this_address , user = this_login , passwd = this_password , timeout = this_timeout ) except (ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e: err_msg = e.args[0] except socket.error, msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) except gui_common.runing_errors,why: err_msg = str(why) if err_msg is not None: gui_common.save_error( this_msg = "ftp_remove(%s) : Unable to open a FTP connection (login: %s, password: %s) : %s"%(this_address,this_login,this_password,err_msg) , this_file_name = __file__ , this_mail_subject = "%s: %s"%(_ftp_error_tag,err_msg) ) return False if this_debug_mode is True: # Set FTP debug trace level # 0: no debugging output (default) # 1: print commands and responses but not body text etc. # 2: also print raw lines read and sent before stripping CR/LF ftp_id.set_debuglevel( level = 1 ) else: # The default value ftp_id.set_debuglevel( level = 0 ) # Go on if this_dst_path is None : # Take the current remote directory as default one try: this_dst_path = ftp_id.pwd() except: gui_common.save_error( this_msg = "ftp_remove(%s) : Unable to obtain the current FTP remote directory"%(this_address) , this_object_class = ftp_id , this_file_name = __file__ , this_mail_subject = _ftp_error_tag ) return False # Go result = recursive_ftp_remove(this_dst_path) # Close the FTP connection _ftp_close_connection( this_ftp_id = ftp_id) # Done return result # ------------------------- # Function : ftp_download() # ------------------------- def ftp_download( this_address = "1.2.3.4" , this_login = "dummy_login" , this_password = "dummy_down_password" , this_timeout = gui_common.ftp_timeout_s , this_src_path = None , this_dst_path = None , this_nickname = None , this_check_size = False , this_max_connection = None , this_remote_directory_flag = None , this_debug_mode = False ): """ Download FTP a file or a directory and its content. Use recursivity when a directory is found. @param this_address: address of the server @type this_address: ascii @param this_login: remote user login @type this_login: ascii @param this_password: remote user password @type this_password: ascii @param this_timeout: timeout in seconds for blocking operations like the connection attempt @type this_timeout: real @param this_src_path: server source directory or file path. Wildcard '*' is allowed on the basename. @type this_src_path: ascii string @param this_dst_path: local destination directory @type this_dst_path: ascii string @param this_user_conf: user configuration @type this_user_conf: gui_user.configuration ident @param this_check_size: True = Check sent and downloaded file size, False = No check @type this_check_size: boolean @param this_max_connection: [optional] max connection number @type this_max_connection: integer @param this_remote_directory_flag: False = remote file, True = remote directory, None = to be determined @type this_remote_directory_flag: boolean @return: True : transfert done / False : error @rtype: boolean """ if this_debug_mode is True: msg = "ftp_download(%s) : from %s to %s, login: %s, pass: %s"%(this_address,this_src_path,this_dst_path,this_login,this_password) gui_common.save_trace( this_msg = msg , this_object_class = None , this_file_name = __file__ ) err_msg = None dummy = os.path.basename(this_src_path) if "*" in dummy: # It's a file or a directory with a wildcard tag_list = [tag_idx for tag_idx in dummy.split("*") if tag_idx != ""] src_path = os.path.dirname(this_src_path) if src_path == "": # No directory path, use the current one ! src_path = os.curdir else: # It should be a directory name or a file name tag_list = [] src_path = this_src_path # FTP or SFTP ? sftp_in_use = gui_common.get_sftp_usage( this_ip_address = this_address) if this_max_connection is None: # Get the maximal allowed connection number max_expected_connections_number = _ftp_get_max_connection( this_address = this_address) else: # Use the wanted user connection number max_expected_connections_number = this_max_connection ftp_cnx_info_list = _ftp_open_many_connection( this_address = this_address , this_login = this_login , this_password = this_password , this_timeout = this_timeout , this_upload = False , this_max_connection = max_expected_connections_number , this_user_remote_dir = src_path , this_nickname = this_nickname , this_check_size = this_check_size , this_use_sftp = gui_common.get_sftp_usage( this_ip_address = this_address) , this_debug_mode = this_debug_mode ) if ftp_cnx_info_list == []: # No opened connection on this server return False # Use the first connection for me. It will be closed by the slave thread. (file_manager_id, ftp_id, _ftp_thread_id) = ftp_cnx_info_list[0] # Go down to work ! if this_dst_path is None : # Take the current source directory as default one this_dst_path = os.getcwd() if this_src_path is None : # Take the current remote directory as default one ;-) try: this_src_path = ftp_id.pwd() except (ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e: err_msg = e.args[0] except socket.error, msg: err_msg = gui_common.get_socket_error_msg( this_error = msg) except: err_msg = "Unable to obtain the current FTP remote directory" if err_msg is not None: gui_common.save_error( this_msg = err_msg , this_object_class = ftp_id , this_file_name = __file__ , this_mail_subject = _ftp_error_tag ) # Close the FTP connections _ftp_close_connection_list( this_list = ftp_cnx_info_list) del file_manager_idx return False else: # Use POSIX name format this_src_path = this_src_path.replace(os.sep,_ftp_posix_separator) # Get remote files & directories remote_file_list = _ftp_get_remote_info( this_ftp_id = ftp_id , this_ftp_tag = this_src_path , this_tag_selector = tag_list ) if remote_file_list == []: # Nothing to transfert _ftp_close_connection_list( this_list = ftp_cnx_info_list) del ftp_cnx_info_list gui_common.garbage_collector_start() return True # Determine the src&dest files & directories local_dir_list = [] for file_info_idx in remote_file_list: try: (remote_file_name, is_directory, file_size) = file_info_idx except gui_common.runing_errors,why: gui_common.save_error( this_msg = "unexpected file_info_idx format: %s, %s"%(str(file_info_idx),str(why)) , this_file_name = __file__ , this_mail_subject = _ftp_error_tag ) _ftp_close_connection_list( this_list = ftp_cnx_info_list) gui_common.garbage_collector_start() return False real_dst_path = this_dst_path+os.sep+remote_file_name # FTP format: Linux --> Windows real_dst_path = real_dst_path.replace('/',os.sep) if is_directory is True: # Remote directory local_dir_list.append(real_dst_path) else: # Remote File file_info = (remote_file_name, real_dst_path, file_size) file_manager_id.add_waiting( this_scenario_name = file_info , this_ignore_duplication_warning = True ) # Next file_info_idx continue if local_dir_list != []: # Build local directories gui_common.build_directories( this_directory_list = local_dir_list) # Retreive back the files now ! for ftp_cnx_info in ftp_cnx_info_list: (file_manager_id, ftp_id, ftp_thread_id) = ftp_cnx_info ftp_thread_id.start() # Next ftp_cnx_info continue # Wait end of transfert result = True for ftp_cnx_info in ftp_cnx_info_list: (file_manager_id, ftp_id, ftp_thread_id) = ftp_cnx_info ftp_thread_id.join() result = result and ftp_thread_id.result # Next ftp_cnx_info continue _ftp_close_connection_list( this_list = ftp_cnx_info_list) del ftp_cnx_info_list # Check if the transfert has been done correctly remaining_file_list = file_manager_id.get_running_list(this_name_only = True) if remaining_file_list != []: # Some files have not been transfered subject = "%s Some files have not been downloaded to %s from %s"%( gui_common.ftp_download_marker , gui_common.get_ip_address() , this_address ) file_list = [this_src_path,] file_list += [os.path.basename(full_src_file) for full_src_file in remaining_file_list if isinstance(full_src_file,(str,unicode)) is True] gui_common.send_mail( this_subject = subject , this_content = gui_common.char_line_feed.join([src_file for (src_file, dest_file, file_size) in remaining_file_list]) , this_receiver_email_addr = gui_common.administrator_email ) # Done gui_common.garbage_collector_start() return result # ----------------------- # Function : ftp_upload() # ----------------------- def ftp_upload( this_address = "1.2.3.4" , this_login = "dummy_up_login" , this_password = "dummy_up_password" , this_timeout = gui_common.ftp_timeout_s , this_src_path = None , this_dst_path = None , this_unwanted_tag_list = [gui_common.svn_dir_name,] , this_nickname = None , this_check_size = True , this_max_connection = None , this_debug_mode = False ): """ Upload FTP a file or a directory and its content. Use recursivity when a directory is found. @param this_address: address of the server @type this_address: ascii @param this_login: remote user login @type this_login: ascii @param this_password: remote user password @type this_password: ascii @param this_timeout: timeout in seconds for blocking operations like the connection attempt @type this_timeout: real @param this_src_path: source path name (file or directory). A wildcard '*' is allowed inside a file name (not the directory one). @type this_src_path: ascii string @param this_dst_path: server destination path name @type this_dst_path: ascii string @param this_unwanted_tag_list: do not upload files/directory whose name is present @type this_unwanted_tag_list: list of ascii string @param this_debug_mode: True = with debug mode, False = silent @type this_debug_mode: boolean @return: True : transfert done / False : error @rtype: boolean @note: - It is not possible to create a new remote directory whose name is different from the local one. """ # -------------------------------------- # Private: recursive_build_upload_list() # -------------------------------------- def recursive_build_upload_list( this_src_path , this_dst_path , this_unwanted_tag_list , this_recursiv_call = False ): """ local function for recursivity upload calls @param this_src_path: source path name (file or directory). @type this_src_path: ascii string @param this_dst_path: server destination directory @type this_dst_path: ascii string @param this_recursiv_call: True = recursiv call, the directory is already taken into account, False = first call @type this_recursiv_call: boolean @return: True : transfert done / False : error @rtype: boolean Note: Use binary transfert for ASCII and binary files tranfert ;-) """ dst_path = copy.copy(this_dst_path) err_msg = None # Be careful with the first call... ;-) if os.path.isdir(this_src_path) is True: if this_recursiv_call is False: base_name = os.path.basename(this_src_path) # Use the POSIX separator for remote files ! dst_path += os.sep+base_name dst_path = dst_path.replace(os.sep,_ftp_posix_separator) # Get the directory list src_dir_list = os.listdir(this_src_path) else: if os.path.isfile(this_src_path) is True: # Only one file to upload src_dir_list = [os.path.basename(this_src_path),] this_src_path = os.path.dirname(this_src_path) else: # Do not follow links, ignore it return True # Exam the result list for src_file in src_dir_list: keep_file = True for unwanted_tag_idx in this_unwanted_tag_list: if unwanted_tag_idx in src_file: # Not wanted ! keep_file = False break # Next unwanted_tag_idx continue if keep_file is False: # Next ! continue full_src_file = this_src_path+os.sep+src_file # Use the POSIX separator for remote files ! full_dest_file = dst_path+os.sep+src_file full_dest_file = full_dest_file.replace(os.sep,_ftp_posix_separator) if os.path.isfile(full_src_file) is True: # Add the file for transfert try: file_host_size = os.path.getsize(full_src_file) except gui_common.runing_errors,why: # Should never occure err_msg = str(why) except: # Idem ! err_msg = "os.path.getsize(%s)"%(full_src_file) else: info = (full_src_file, full_dest_file, file_host_size) file_manager_id.add_waiting( this_scenario_name = info) if err_msg is not None: gui_common.save_warning( this_msg = "os.path.getsize(%s): %s"%(full_src_file,err_msg) , this_object_class = ftp_id , this_file_name = __file__ , this_caller_name = "recursive_build_upload_list()" ) return False else: if os.path.isdir(full_src_file) is True: new_dst_path = dst_path+_ftp_posix_separator+os.path.basename(full_src_file) # Do a recursive call if recursive_build_upload_list( this_src_path = full_src_file , this_dst_path = new_dst_path , this_unwanted_tag_list = this_unwanted_tag_list , this_recursiv_call = True ) is False: return False # Next src file continue return True # # Main body of ftp_upload() # if isinstance(this_dst_path,(str,unicode)) is False: subject = "%s Dest path is not an ASCII string '%s'"%( gui_common.ftp_upload_marker , str(this_dst_path) ) current_thread_id = threading.currentThread() content = "%s\n%s"%(str(current_thread_id),gui_common.get_thread_stack()) gui_common.send_mail( this_subject = subject , this_content = content , this_receiver_email_addr = gui_common.administrator_email ) return False if isinstance(this_src_path,(str,unicode)) is False: subject = "%s Src path is not an ASCII string '%s'"%( gui_common.ftp_upload_marker , str(this_src_path) ) current_thread_id = threading.currentThread() content = "%s\n%s"%(str(current_thread_id),gui_common.get_thread_stack()) gui_common.send_mail( this_subject = subject , this_content = content , this_receiver_email_addr = gui_common.administrator_email ) return False if this_debug_mode is True: gui_common.save_trace( this_msg = "ftp_upload(%s) from %s to %s, login: %s, pass: %s"%(this_address,this_src_path,this_dst_path,this_login,this_password) , this_object_class = None , this_file_name = __file__ , this_caller_name = "ftp_upload()" ) # FTP or SFTP ? sftp_in_use = gui_common.get_sftp_usage( this_ip_address = this_address) # Be carreful when transmiting one file if os.path.isfile(this_src_path) is True: # Extract remote parameters (UNIX path format...) path_list = this_dst_path.replace(os.sep,_ftp_posix_separator).split(_ftp_posix_separator) user_remote_dir = _ftp_posix_separator.join(path_list[:-1]) # One file , one upload needed max_expected_connections_number = 1 else: # The dst path is a POSIX directory user_remote_dir = this_dst_path.replace(os.sep,_ftp_posix_separator) if this_max_connection is None: max_expected_connections_number = _ftp_get_max_connection( this_address = this_address) else: max_expected_connections_number = this_max_connection # Open FTP connections ftp_cnx_info_list = _ftp_open_many_connection( this_address = this_address , this_login = this_login , this_password = this_password , this_timeout = this_timeout , this_upload = True , this_user_remote_dir = this_dst_path , this_max_connection = max_expected_connections_number , this_debug_mode = this_debug_mode , this_nickname = this_nickname , this_check_size = this_check_size , this_use_sftp = sftp_in_use ) if ftp_cnx_info_list == []: # No open connection on this server return False # Use the first connection to obtain file names. It will be closed by the slave thread. (file_manager_id, ftp_id, _ftp_thread_id) = ftp_cnx_info_list[0] err_msg = None # Go on src_directory_with_wildcard = '*' in this_src_path if this_dst_path is None: # Use the default one this_dst_path = _ftp_thread_id.user_remote_dir # This_dst_path may be as well as a file or a directory name. if src_directory_with_wildcard is True: # A wildcard has been given, Take the remote path as is ! # Multiple targets target_list = glob.glob(this_src_path) else: # No Wildcard: only one target target_list = [this_src_path,] # Avoid to have // directory inside recursive_build_upload_list() function if user_remote_dir == _ftp_posix_separator: user_remote_dir = "" # Check if the remote base directory exists or not if src_directory_with_wildcard is True: src_initial_path = this_src_path.split("*")[0] else: src_initial_path = this_src_path # Run my hen ! for target_file in target_list: if recursive_build_upload_list( this_src_path = target_file , this_dst_path = user_remote_dir , this_unwanted_tag_list = this_unwanted_tag_list ) is not True: _ftp_close_connection_list( this_list = ftp_cnx_info_list) del ftp_cnx_info_list gui_common.garbage_collector_start() return False # Next target_file continue # Start the file transferts now ! for ftp_cnx_info in ftp_cnx_info_list: (file_manager_idx, ftp_idx, ftp_thread_idx) = ftp_cnx_info ftp_thread_idx.start() # Next ftp_cnx_info continue # Wait end of transfert for ftp_cnx_info in ftp_cnx_info_list: (file_manager_idx, ftp_idx, ftp_thread_idx) = ftp_cnx_info ftp_thread_idx.join() # Next ftp_cnx_info continue # Check if the transfert has been done correctly remaining_file_list = file_manager_id.get_running_list(this_name_only = True) result = (remaining_file_list == []) if result is False: # Some files have not been transfered subject = "%s Some files have not been uploaded to %s"%( gui_common.ftp_upload_marker , this_address ) # file_list = ["Files not uploaded:\n",] for file_to_transfert_info in remaining_file_list: try: (full_src_file, full_dest_file, file_size) = file_to_transfert_info except: dummy = "<<>> "+str(file_to_transfert_info) else: dummy = "full_src_file: %s\nfull_dest_file: %s\nfile_size: %d\n"%file_to_transfert_info file_list.append(dummy) continue gui_common.send_mail( this_subject = subject , this_content = gui_common.char_line_feed.join(file_list) , this_receiver_email_addr = gui_common.administrator_email ) # Done _ftp_close_connection_list( this_list = ftp_cnx_info_list) del ftp_cnx_info_list gui_common.garbage_collector_start() return result # -------------------------------- # Function : ftp_parallel_upload() # -------------------------------- def ftp_parallel_upload( this_address_list = [] , this_login = "dummy_login" , this_password = "dummy_par_up_password" , this_timeout = gui_common.ftp_timeout_s , this_src_path = None , this_dst_path = None , this_cleanup_flag = False , this_debug_mode = False ): """ Upload FTP simultaneously on miscellaneous remote servers. @param this_address_list: list of remote address @type this_address_list: list @param this_login: remote user login @type this_login: ascii @param this_password: remote user password @type this_password: ascii @param this_timeout: timeout in seconds for blocking operations like the connection attempt @type this_timeout: real @param this_src_path: local source directory @type this_src_path: ascii string @param this_dst_path: server destination directory @type this_dst_path: ascii string @param this_cleanup_flag: force to clean up the destination directory (True) or not (False) @type this_cleanup_flag: boolean @param this_debug_mode: current debug mode @type this_debug_mode: boolean @return: updated server list without any error @rtype: list """ if isinstance(this_dst_path,(str,unicode)) is False: subject = "%s Dest path is not an ASCII string '%s'"%( gui_common.ftp_upload_marker , str(this_dst_path) ) current_thread_id = threading.currentThread() content = "%s\n%s"%(str(current_thread_id),gui_common.get_thread_stack()) gui_common.send_mail( this_subject = subject , this_content = content , this_receiver_email_addr = gui_common.administrator_email ) return False if isinstance(this_src_path,(str,unicode)) is False: subject = "%s Src path is not an ASCII string '%s'"%( gui_common.ftp_upload_marker , str(this_src_path) ) current_thread_id = threading.currentThread() content = "%s\n%s"%(str(current_thread_id),gui_common.get_thread_stack()) gui_common.send_mail( this_subject = subject , this_content = content , this_receiver_email_addr = gui_common.administrator_email ) return False updated_server_list = [] son_id_list = [] ip_address = gui_common.get_ip_address() for host_address in this_address_list: if host_address != ip_address: # Update this server son_name = "FTP_Upload_"+host_address son_id = _ftp_thread( this_address = host_address , this_login = this_login , this_password = this_password , this_timeout = this_timeout , this_src_path = this_src_path , this_dst_path = this_dst_path , this_direction= _upload_cmd , this_cleanup_flag = this_cleanup_flag , this_debug_mode = this_debug_mode ) son_id_list.append(son_id) son_id.setName(son_name) son_id.start() # Next host_address continue # Wait the end of my sons for son_id in son_id_list: son_id.join() if son_id.result is True: updated_server_list.append(son_id.address) # Next son_id continue # Done del son_id_list gui_common.garbage_collector_start() return updated_server_list # ---------------------------------- # Function : ftp_parallel_download() # ---------------------------------- def ftp_parallel_download( this_address_list = [] , this_login = "dummy_login" , this_password = "dummy_par_down_password" , this_timeout = gui_common.ftp_timeout_s , this_src_path = None , this_dst_path = None , this_debug_mode = False ): """ Download FTP simultaneously from miscellaneous remote servers. @param this_address_list: list of remote address @type this_address_list: list @param this_login: remote user login @type this_login: ascii @param this_password: remote user password @type this_password: ascii @param this_timeout: timeout in seconds for blocking operations like the connection attempt @type this_timeout: real @param this_src_path: local source directory @type this_src_path: ascii string @param this_dst_path: server destination directory @type this_dst_path: ascii string @param this_debug_mode: current debug mode @type this_debug_mode: boolean @return: downloaded server list without any error @rtype: list """ downloaded_server_list,son_id_list = [],[] ip_address = gui_common.get_ip_address() for host_address in this_address_list: if host_address != ip_address: # Download from this server son_name = "FTP_Download_"+host_address son_id = _ftp_thread( this_address = host_address , this_login = this_login , this_password = this_password , this_timeout = this_timeout , this_src_path = this_src_path , this_dst_path = this_dst_path , this_direction= _download_cmd , this_cleanup_flag = False , this_debug_mode = this_debug_mode ) son_id_list.append(son_id) son_id.setName(son_name) son_id.start() # Next host_address continue # Wait the end of my sons for son_id in son_id_list: son_id.join() if son_id.result is True: downloaded_server_list.append(son_id.address) # Next son_id continue del son_id_list gui_common.garbage_collector_start() return downloaded_server_list # ---------------------------------------------------- # If the module is run alone (i.e. for debug) : Main() # ---------------------------------------------------- if __name__ == '__main__': # # Unitary Test # # -------------------------------- # SFTP upload error ON server side # previous upload Ok # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> TYPE I # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 200 Type set to I # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> PASV # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 227 Entering Passive Mode (135,120,162,6,213,254) # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> STOR Introduction_PUCch_F2_Model.tex # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 150 Opening data channel for file upload to server of "/Introduction_PUCch_F2_Model.tex" # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> SSL connection for data connection established # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 226 Successfully transferred "/Introduction_PUCch_F2_Model.tex" # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> SIZE Introduction_PUCch_F2_Model.tex # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 213 3155 # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> TYPE I # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 200 Type set to I # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> PASV # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 227 Entering Passive Mode (135,120,162,6,236,58) # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> STOR Introduction_PUCch_F1_Model.tex # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 150 Opening data channel for file upload to server of "/Introduction_PUCch_F1_Model.tex" # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> SSL connection for data connection established # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 226 Successfully transferred "/Introduction_PUCch_F1_Model.tex" # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> SIZE Introduction_PUCch_F1_Model.tex # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 213 3148 # until here, it's ok ! Same player shoots again... # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> TYPE I # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 200 Type set to I # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> PASV # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 227 Entering Passive Mode (135,120,162,6,200,16) # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> STOR Handle.exe # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 150 Opening data channel for file upload to server of "/Handle.exe" # the problem is comming... # (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 550 can't access file. <<<<<< Instead of: "SSL connection for data connection established" # (000336)30/09/2015 08:57:29 - nemo_pyc (135.238.179.15)> TYPE I # (000336)30/09/2015 08:57:29 - nemo_pyc (135.238.179.15)> 200 Type set to I # (000336)30/09/2015 08:57:59 - nemo_pyc (135.238.179.15)> 3DG} # (000336)30/09/2015 08:58:04 - nemo_pyc (135.238.179.15)> 'Y'jQ~. # 0 3DG~ # (000336)30/09/2015 08:58:04 - nemo_pyc (135.238.179.15)> 500 Syntax error, command unrecognized. # (000336)30/09/2015 08:58:09 - nemo_pyc (135.238.179.15)> ͈!xdA:\ 3DG # (000336)30/09/2015 08:58:09 - nemo_pyc (135.238.179.15)> 500 Syntax error, command unrecognized. # (000336)30/09/2015 08:58:09 - nemo_pyc (135.238.179.15)> L._8; # (000336)30/09/2015 08:58:09 - nemo_pyc (135.238.179.15)> 500 Syntax error, command unrecognized. # (000336)30/09/2015 08:58:09 - nemo_pyc (135.238.179.15)> PzR # (000336)30/09/2015 08:58:09 - nemo_pyc (135.238.179.15)> 500 Syntax error, command unrecognized. # (000336)30/09/2015 08:58:14 - nemo_pyc (135.238.179.15)> 3DG # (000336)30/09/2015 08:58:18 - nemo_pyc (135.238.179.15)> ˒_SYS 3DG # (000336)30/09/2015 08:58:18 - nemo_pyc (135.238.179.15)> 500 Syntax error, command unrecognized. # (000336)30/09/2015 08:58:23 - nemo_pyc (135.238.179.15)> l 8]Ԝkj¾ 3DG # (000336)30/09/2015 08:58:23 - nemo_pyc (135.238.179.15)> 500 Syntax error, command unrecognized. # (000336)30/09/2015 08:58:27 - nemo_pyc (135.238.179.15)> X#Vb]$ 3DG # (000336)30/09/2015 08:58:27 - nemo_pyc (135.238.179.15)> 500 Syntax error, command unrecognized. # (000336)30/09/2015 08:58:32 - nemo_pyc (135.238.179.15)> ^bVY"7 3DG # (000336)30/09/2015 08:58:32 - nemo_pyc (135.238.179.15)> 500 Syntax error, command unrecognized. # (000336)30/09/2015 08:58:37 - nemo_pyc (135.238.179.15)> nBgI 'Im&FP 3DG # (000336)30/09/2015 08:58:37 - nemo_pyc (135.238.179.15)> 500 Syntax error, command unrecognized. # (000336)30/09/2015 08:58:41 - nemo_pyc (135.238.179.15)> OrmZQ} 3DG # (000336)30/09/2015 08:58:41 - nemo_pyc (135.238.179.15)> 500 Syntax error, command unrecognized. # (000336)30/09/2015 08:58:41 - nemo_pyc (135.238.179.15)> +Y|ғհAIw # (000336)30/09/2015 08:58:41 - nemo_pyc (135.238.179.15)> 500 Syntax error, command unrecognized. # -------------------------------- # SFTP upload OK ON Client side # +--- 135.238.179.15 - Wed Sep 30 09:07:27 2015 - FRVILN0H300423 # | Process : <_MainProcess(MainProcess, started)> # | Thread : <_ftp_slave_thread(FTP_135.120.162.9_Upload_1, started 3752)> # | File : gui_ftp.py # | Comment : <<>> C:\Nemo_Pyc\Handle.exe --> nemo_tmp/dst_dir/Nemo_Pyc/Handle.exe # +--- # *cmd* 'TYPE I' # *resp* '200 Type set to I' # *cmd* 'PASV' # *resp* '227 Entering Passive Mode (135,120,162,9,226,12)' # *cmd* 'STOR Handle.exe' # *resp* '150 Opening data channel for file upload to server of "/nemo_tmp/dst_dir/Nemo_Pyc/Handle.exe"' # *resp* '226 Successfully transferred "/nemo_tmp/dst_dir/Nemo_Pyc/Handle.exe"' # *cmd* 'SIZE Handle.exe' # *resp* '213 536256' # ---------------------------------- def test_ftp_upload(): user_conf = gui_user.configuration() user_conf.debug_mode = False test_dst_dir = "Nemo_Tmp/up" if gui_common.get_ip_address() == user_conf.get_selected_dispatcher(): # Test dispatcher - labo remote_ip = "135.120.162.1" src_dir = "C:\\Nemo_Tmp\\up" # Send one dir print "=== ftp_upload: to %s ==="%(remote_ip) ftp_upload( this_address = remote_ip , this_login = gui_common.main_login , this_password = gui_common.main_passwd_default , this_timeout = gui_common.ftp_timeout_s , this_src_path = src_dir , this_dst_path = test_dst_dir , this_debug_mode = user_conf.debug_mode ) print "=== ftp_upload: one dir/* to %s ==="%(remote_ip) dir_name = src_dir+os.sep+"*" ftp_upload( this_address = remote_ip , this_login = gui_common.main_login , this_password = gui_common.main_passwd_default , this_timeout = gui_common.ftp_timeout_s , this_src_path = dir_name , this_dst_path = test_dst_dir , this_debug_mode = user_conf.debug_mode ) else: # Test laptop - labo server_list = [ "135.120.162.1" , "135.120.162.2" , "135.120.162.3" , "135.120.162.4" , "135.120.162.5" , "135.120.162.6" , "135.120.162.7" , "135.120.162.8" , "135.120.162.9" ] remote_ip = user_conf.get_selected_dispatcher() src_dir = "C:\\Nemo_Tests\\BLANQUI1.3\\Nemo2\\python\\sce" # Send in parallel print "=== ftp_parallel_upload: %s to servers ==="%(src_dir) ftp_parallel_upload( this_address_list = server_list , this_login = gui_common.main_login , this_password = gui_common.main_passwd_new , this_timeout = gui_common.ftp_timeout_s , this_src_path = src_dir , this_dst_path = test_dst_dir , this_cleanup_flag = True , this_debug_mode = False ) # Send some directories print "=== ftp_upload: %s/* to %s ==="%(src_dir, remote_ip) ftp_upload( this_address = remote_ip , this_login = gui_common.main_login , this_password = gui_common.main_passwd_default , this_timeout = gui_common.ftp_timeout_s , this_src_path = src_dir+os.sep+"*" , this_dst_path = test_dst_dir , this_debug_mode = user_conf.debug_mode ) # Send one file file_name = user_conf.ftp_pyc_directory+os.sep+"gui_ftp.pyc" print "=== ftp_upload: %s to %s ==="%(file_name,remote_ip) ftp_upload( this_address = remote_ip , this_login = gui_common.main_login , this_password = gui_common.main_passwd_default , this_timeout = gui_common.ftp_timeout_s , this_src_path = file_name , this_dst_path = test_dst_dir , this_debug_mode = user_conf.debug_mode ) # Send one dir dir_name = user_conf.ftp_pyc_directory+os.sep+"*" print "=== ftp_upload: one dir to %s ==="%(remote_ip) ftp_upload( this_address = remote_ip , this_login = gui_common.main_login , this_password = gui_common.main_passwd_default , this_timeout = gui_common.ftp_timeout_s , this_src_path = dir_name , this_dst_path = test_dst_dir , this_debug_mode = user_conf.debug_mode ) return # ---------------------------------- def test_ftp_download(): # Download test user_conf = gui_user.configuration() user_conf.debug_mode = False user_remote_dir = "BLANQUI1.3" if gui_common.get_ip_address() == user_conf.get_selected_dispatcher(): # Test dispatcher - labo remote_ip = "135.120.162.1" src_dir = "Nemo_Tmp\\up" dest_dir = "c:\\Nemo_Tmp\\up1" try: os.mkdir(dest_dir) except: pass print "=== ftp_download ===" ftp_download( this_address = dest_ip_addr , this_login = user_conf.ftp_login , this_password = user_conf.ftp_password , this_timeout = gui_common.ftp_timeout_s , this_src_path = user_remote_dir , this_dst_path = dest_dir , this_debug_mode = user_conf.debug_mode ) else: # Test laptop - labo dest_dir = "Z:"+os.sep+"ftp_down" dest_ip_addr = user_conf.get_selected_dispatcher() if os.path.exists(dest_dir) is True: if gui_common.remove_directory( this_target = dest_dir) is not True: print "ERROR" print "=== ftp_parallel_download ===" server_list = [ "135.120.162.1" , "135.120.162.2" , "135.120.162.3" , "135.120.162.4" , "135.120.162.5" , "135.120.162.6" , "135.120.162.7" , "135.120.162.8" , "135.120.162.9" ] ftp_parallel_download( this_address_list = server_list , this_login = gui_common.main_login , this_password = gui_common.main_passwd_new , this_timeout = gui_common.ftp_timeout_s , this_src_path = user_remote_dir , this_dst_path = dest_dir , this_debug_mode = False ) print "=== ftp_download ===" ftp_download( this_address = dest_ip_addr , this_login = user_conf.ftp_login , this_password = user_conf.ftp_password , this_timeout = gui_common.ftp_timeout_s , this_src_path = user_remote_dir , this_dst_path = dest_dir , this_debug_mode = user_conf.debug_mode ) return # ---------------------------------- def test_system_info(): dest_dir= "D:\\dummy\\Logs" user_conf = gui_user.configuration() user_conf.debug_mode = False ip_address = user_conf.get_selected_dispatcher() ftp_cnx_info_list = _ftp_open_many_connection( this_address = ip_address , this_login = gui_common.main_login , this_password = gui_common.main_passwd_default , this_timeout = 10.0 , this_upload = False , this_user_remote_dir = None , this_max_connection = 1 , this_check_size = False , this_use_sftp = gui_common.get_sftp_usage( this_ip_address = ip_address) ) (file_manager_id, ftp_id, _ftp_thread_id) = ftp_cnx_info_list[0] # Get remote server info (Filezilla Server) answer = _ftp_get_server_info( this_ftp_id = ftp_id) (system_info, feature_list) = answer print "System :" + system_info print "Feature:" for feature_idx in feature_list: print " "+feature_idx continue # Get detailed remote directory info (Filezilla Server) answer = _ftp_get_directory_detail( this_ftp_id = ftp_id , this_dir = "Nemo_Tests" ) for idx in answer: (directory_flag, size, year, month, date, hour_minute, file) = idx msg = "%s %8d %d.%s.%02d.%s %s"%idx print msg continue _ftp_close_connection( this_ftp_id = ftp_id) return # ---------------------------------- def test_sftp_download( this_address = "135.120.162.9" , this_login = gui_common.main_login , this_password = gui_common.main_passwd_default , this_timeout = 6000.0 # 100 mn for debugging ! ): ftp_download( this_address = this_address , this_login = gui_common.main_login , this_password = gui_common.main_passwd_default , this_timeout = this_timeout , this_src_path = "nemo_pyc" , this_dst_path = gui_common.nemo_all_result_directory , this_check_size = True , this_max_connection = None , this_debug_mode = False # True ) return # ---------------------------------- def test_sftp_upload( this_address = "135.120.162.9" , this_login = gui_common.main_login , this_password = gui_common.main_passwd_default , this_timeout = 6000.0 # 100 mn for debugging ! ): src_dir = "D:\\z_Code\\NEMO2_2015_03_24_EXTENDEDCP_UPDATES_3062\\main\\Automatic_Launcher" src_dir = "C:\\Nemo_Pyc" ftp_upload( this_address = this_address , this_login = gui_common.main_login , this_password = gui_common.main_passwd_default , this_timeout = this_timeout , this_src_path = src_dir , this_dst_path = "nemo_tmp\\dst_dir" , this_check_size = True , this_max_connection = None , this_debug_mode = False # True ) return # ---------------------------------- start_time = time.time() test_sftp_upload() #test_sftp_download() #test_ftp_upload() #test_ftp_download() #test_system_info() msg = " elapsed time: %s"%(gui_common.elapsed_time_ascii( this_start_time = start_time)) gui_common.wait_CR_pressed( this_message = msg) #improve_software_pref()