some paramiko dependant tests fail reproducible

[frispete]

Hi,

FYI: all current versions of paramiko (I tested 1.15.1, 1.16.0, and 2.0.0) with Python 2.7 trigger failures in the fs testsuite:

[   72s] ____________________ TestSFTPFS.test_cases_in_separate_dir____________________
[   72s] 
[   72s] self = <fs.tests.test_expose.TestSFTPFS testMethod=test_cases_in_separate_dirs>
[   72s] 
[   72s]     def test_cases_in_separate_dirs(self):
[   72s]         class TestCases_in_subdir(self.__class__, unittest.TestCase):
[   72s]             """Run all testcases against a subdir of self.fs"""
[   72s]             def __init__(this, subdir):
[   72s]                 super(TestCases_in_subdir, this).__init__("test_listdir")
[   72s]                 this.subdir = subdir
[   72s]                 for meth in dir(this):
[   72s]                     if not meth.startswith("test_"):
[   72s]                         continue
[   72s]                     if meth in self._dont_retest:
[   72s]                         continue
[   72s]                     if not hasattr(FSTestCases, meth):
[   72s]                         continue
[   72s]                     if self.fs.exists(subdir):
[   72s]                         self.fs.removedir(subdir, force=True)
[   72s]                     self.assertFalse(self.fs.isdir(subdir))
[   72s]                     self.assertTrue(self.fs.isdir("/"))
[   72s]                     self.fs.makedir(subdir)
[   72s]                     self._yield()
[   72s]                     getattr(this, meth)()
[   72s]     
[   72s]             @property
[   72s]             def fs(this):
[   72s]                 return self.fs.opendir(this.subdir)
[   72s]     
[   72s]             def check(this, p):
[   72s]                 return self.check(pathjoin(this.subdir, relpath(p)))
[   72s]     
[   72s]         def thread1():
[   72s]             TestCases_in_subdir("thread1")
[   72s]     
[   72s]         def thread2():
[   72s]             TestCases_in_subdir("thread2")
[   72s]     
[   72s]         def thread3():
[   72s]             TestCases_in_subdir("thread3")
[   72s] >       self._runThreads(thread1, thread2, thread3)
[   72s] 
[   72s] fs/tests/__init__.py:1074: 
[   72s] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
[   72s] fs/tests/__init__.py:962: in runThread
[   72s]     func()
[   72s] fs/tests/__init__.py:1067: in thread1
[   72s]     TestCases_in_subdir("thread1")
[   72s] fs/tests/__init__.py:1057: in __init__
[   72s]     getattr(this, meth)()
[   72s] fs/tests/__init__.py:534: in test_infokeys
[   72s]     self.assertEqual(self.fs.getinfokeys('info.txt', k), {k: v})
[   72s] E   AssertionError: {'asbytes': <bound method SFTPAttributes.asbytes of <SFTPAttributes: [ size=13 m [truncated]... != {'asbytes': <bound method SFTPAttributes.asbytes of <SFTPAttributes: [ size=13 m [truncated]...
[   72s] E     {'asbytes': <bound method SFTPAttributes.asbytes of <SFTPAttributes: [ size=13 mode=0100777 atime=1462790360 mtime=1462790360 ]>>}

[   72s]______________________TestSFTPFS.test_concurrent_copydir______________________
[   72s] 
[   72s] self = <fs.tests.test_expose.TestSFTPFS testMethod=test_concurrent_copydir>
[   72s] 
[   72s]     def test_concurrent_copydir(self):
[   72s]         self.fs.makedir("a")
[   72s]         self.fs.makedir("a/b")
[   72s]         self.fs.setcontents("a/hello.txt", b("hello world"))
[   72s]         self.fs.setcontents("a/guido.txt", b("is a space alien"))
[   72s]         self.fs.setcontents("a/b/parrot.txt", b("pining for the fiords"))
[   72s]     
[   72s]         def copydir():
[   72s]             self._yield()
[   72s]             self.fs.copydir("a", "copy of a")
[   72s]     
[   72s]         def copydir_overwrite():
[   72s]             self._yield()
[   72s]             self.fs.copydir("a", "copy of a", overwrite=True)
[   72s]         # This should error out since we're not overwriting
[   72s]         self.assertRaises(
[   72s]             DestinationExistsError, self._runThreads, copydir, copydir)
[   72s]         self.assert_(self.fs.isdir('a'))
[   72s] >       self.assert_(self.fs.isdir('a'))
[   72s] E       AssertionError: False is not true
[   72s] 
[   72s] fs/tests/__init__.py:1140: AssertionError

[   72s]___________________________TestSFTPFS.test_infokeys___________________________
[   72s] 
[   72s] self = <fs.tests.test_expose.TestSFTPFS testMethod=test_infokeys>
[   72s] 
[   72s]     def test_infokeys(self):
[   72s]         test_str = b("Hello, World!")
[   72s]         self.fs.setcontents("info.txt", test_str)
[   72s]         info = self.fs.getinfo("info.txt")
[   72s]         for k, v in info.iteritems():
[   72s] >           self.assertEqual(self.fs.getinfokeys('info.txt', k), {k: v})
[   72s] E           AssertionError: {'asbytes': <bound method SFTPAttributes.asbytes of <SFTPAttributes: [ size=13 m [truncated]... != {'asbytes': <bound method SFTPAttributes.asbytes of <SFTPAttributes: [ size=13 m [truncated]...
[   72s] E             {'asbytes': <bound method SFTPAttributes.asbytes of <SFTPAttributes: [ size=13 mode=0100777 atime=1462790361 mtime=1462790361 ]>>}
[   72s] 
[   72s] fs/tests/__init__.py:534: AssertionError

[   72s] ________________________ TestSFTPFS.test_makedir_winner ________________________
[   72s] 
[   72s] self = <fs.tests.test_expose.TestSFTPFS testMethod=test_makedir_winner>
[   72s] 
[   72s]     def test_makedir_winner(self):
[   72s]         errors = []
[   72s]     
[   72s]         def makedir():
[   72s]             try:
[   72s]                 self.fs.makedir("testdir")
[   72s]             except DestinationExistsError, e:
[   72s]                 errors.append(e)
[   72s]     
[   72s]         def makedir_noerror():
[   72s]             try:
[   72s]                 self.fs.makedir("testdir", allow_recreate=True)
[   72s]             except DestinationExistsError, e:
[   72s]                 errors.append(e)
[   72s]     
[   72s]         def removedir():
[   72s]             try:
[   72s]                 self.fs.removedir("testdir")
[   72s]             except (ResourceNotFoundError, ResourceLockedError), e:
[   72s]                 errors.append(e)
[   72s]         # One thread should succeed, one should error
[   72s] >       self._runThreads(makedir, makedir)
[   72s] 
[   72s] fs/tests/__init__.py:1097: 
[   72s] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
[   72s] fs/tests/__init__.py:962: in runThread
[   72s]     func()
[   72s] fs/tests/__init__.py:1081: in makedir
[   72s]     self.fs.makedir("testdir")
[   72s] fs/base.py:146: in acquire_lock
[   72s]     return func(self, *args, **kwargs)
[   72s] fs/errors.py:257: in wrapper
[   72s]     return func(self,*args,**kwds)
[   72s] fs/sftpfs.py:497: in makedir
[   72s]     stat = self.client.stat(npath)
[   72s] /usr/lib/python2.7/site-packages/paramiko/sftp_client.py:413: in stat
[   72s]     t, msg = self._request(CMD_STAT, path)
[   72s] /usr/lib/python2.7/site-packages/paramiko/sftp_client.py:730: in _request
[   72s]     return self._read_response(num)
[   72s] /usr/lib/python2.7/site-packages/paramiko/sftp_client.py:781: in _read_response
[   72s]     self._convert_status(msg)
[   72s] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
[   72s] 
[   72s] self = <paramiko.sftp_client.SFTPClient object at 0x7f7045cce310>
[   72s] msg = paramiko.Message('\x00\x00\x00\x02\x00\x00\x00\x02\x00\x00\x00\x0cNo such file\x00\x00\x00\x00')
[   72s] 
[   72s]     def _convert_status(self, msg):
[   72s]         """
[   72s]             Raises EOFError or IOError on error status; otherwise does nothing.
[   72s]             """
[   72s]         code = msg.get_int()
[   72s]         text = msg.get_text()
[   72s]         if code == SFTP_OK:
[   72s]             return
[   72s]         elif code == SFTP_EOF:
[   72s]             raise EOFError(text)
[   72s]         elif code == SFTP_NO_SUCH_FILE:
[   72s]             # clever idea from john a. meinel: map the error codes to errno
[   72s] >           raise IOError(errno.ENOENT, text)
[   72s] E           ResourceNotFoundError: Resource not found: None
[   72s] 
[   72s] /usr/lib/python2.7/site-packages/paramiko/sftp_client.py:807: ResourceNotFoundError

[lurch]

96bd56b fixes the infokeys problem, but I haven't yet delved into the concurrency problems.