ftp_utils: New configuration variables and FTP/SFTP ls method update

* Adds new configuration variables for Elsevier SFTP connection - port, sftp

* Adds an addition cleaning to file names on the list of files/folders from ls
  method in ftp_utils.py

Signed-off-by: Wojciech Ziółek <wojciech.ziolek@cern.ch>

Author: Wojciech Ziółek

harvestingkit/contrast_out.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 ##
 ## This file is part of Harvesting Kit.
-## Copyright (C) 2013, 2014 CERN.
+## Copyright (C) 2013, 2014, 2016 CERN.
 ##
 ## Harvesting Kit is free software; you can redistribute it and/or
 ## modify it under the terms of the GNU General Public License as
@@ -99,7 +99,9 @@ def connect(self):
             try:
                 self.ftp = FtpHandler(self.config.ELSEVIER.URL,
                                       self.config.ELSEVIER.LOGIN,
-                                      self.config.ELSEVIER.PASSWORD)
+                                      self.config.ELSEVIER.PASSWORD,
+                                      port = int(self.config.ELSEVIER.PORT),
+                                      sftp = self.config.ELSEVIER.SFTP)
                 self.logger.debug(('Successful connection to the '
                                    'Elsevier server'))
                 return

harvestingkit/elsevier_package.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of Harvesting Kit.
-# Copyright (C) 2013, 2014, 2015 CERN.
+# Copyright (C) 2013, 2014, 2015, 2016 CERN.
 #
 # Harvesting Kit is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License as

harvestingkit/ftp_utils.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 ##
 ## This file is part of Harvesting Kit.
-## Copyright (C) 2014 CERN.
+## Copyright (C) 2014, 2016 CERN.
 ##
 ## Harvesting Kit is free software; you can redistribute it and/or
 ## modify it under the terms of the GNU General Public License as
@@ -195,13 +195,17 @@ def ls(self, folder=''):
             contents = self._sftp_client.listdir()
             files = filter(lambda a: str(self._sftp_client.lstat(a)).split()[0].startswith('-'), contents)
             folders = filter(lambda a: str(self._sftp_client.lstat(a)).split()[0].startswith('d'), contents)
+            files = map(lambda a: ' '.join(a.split()[8:]), files)
+            folders = map(lambda a: ' '.join(a.split()[8:]), folders)
             self._sftp_client.chdir(current_folder)
         else:
             current_folder = self._ftp.pwd()
             self.cd(folder)
             self._ftp.retrlines('LIST', lambda a: contents.append(a))
             files = filter(lambda a: a.split()[0].startswith('-'), contents)
             folders = filter(lambda a: a.split()[0].startswith('d'), contents)
+            files = map(lambda a: ' '.join(a.split()[8:]), files)
+            folders = map(lambda a: ' '.join(a.split()[8:]), folders)
             self._ftp.cwd(current_folder)
         return files, folders
 

user_config.cfg

@@ -2,6 +2,8 @@
 login = empty
 password = empty
 url = empty
+port = 21
+sftp = True
 
 [OXFORD]
 login = empty