From 8646909cffaa82105f4bfb6c31e936e862d7f374 Mon Sep 17 00:00:00 2001 From: Michael Gatto Date: Thu, 29 Aug 2013 16:23:08 -0700 Subject: [PATCH 01/55] Moved tests to own package within the irods package. This seems to be the accepted style within the Python community. It also groups them together better. --- Testing.md | 16 +++++++++ irods/test/Testing.md | 33 +++++++++++++++++++ irods/test/__init__.py | 0 browse_test.py => irods/test/browse_test.py | 9 +++-- coll_test.py => irods/test/coll_test.py | 7 +++- file_test.py => irods/test/file_test.py | 7 +++- message_test.py => irods/test/message_test.py | 11 +++++-- meta_test.py => irods/test/meta_test.py | 7 +++- query_test.py => irods/test/query_test.py | 7 +++- .../test/test_connection.py | 7 +++- 10 files changed, 94 insertions(+), 10 deletions(-) create mode 100644 Testing.md create mode 100644 irods/test/Testing.md create mode 100644 irods/test/__init__.py rename browse_test.py => irods/test/browse_test.py (82%) mode change 100755 => 100644 rename coll_test.py => irods/test/coll_test.py (76%) mode change 100755 => 100644 rename file_test.py => irods/test/file_test.py (69%) mode change 100755 => 100644 rename message_test.py => irods/test/message_test.py (97%) mode change 100755 => 100644 rename meta_test.py => irods/test/meta_test.py (77%) mode change 100755 => 100644 rename query_test.py => irods/test/query_test.py (81%) mode change 100755 => 100644 rename test_connection.py => irods/test/test_connection.py (50%) mode change 100755 => 100644 diff --git a/Testing.md b/Testing.md new file mode 100644 index 0000000..f65ad33 --- /dev/null +++ b/Testing.md @@ -0,0 +1,16 @@ +TESTING NOTES +============= + +| Test | Status | Notes | +| ------------- |-------------|------------| +| browse_test.py | fail | `File "browse_test.py", line 12, in test_get_collection coll = sess.get_collection(path) AttributeError: 'iRODSSession' object has no attribute 'get_collection'`| +| coll_test.py | fail | requires username, password, host; fixable| +| file_test.py | fail |`File "file_test.py", line 9, in obj = sess.get_data_object("/tempZone/home/rods/test1") AttributeError: 'iRODSSession' object has no attribute 'get_data_object'`| +| message_test.py | pass || +| meta_test.py | fail | ` + File "meta_test.py", line 10, in + obj = sess.get_data_object("/tempZone/home/rods/test1") +AttributeError: 'iRODSSession' object has no attribute 'get_data_object'` | +| query_test.py | || +| test_connection.py | || + diff --git a/irods/test/Testing.md b/irods/test/Testing.md new file mode 100644 index 0000000..36530da --- /dev/null +++ b/irods/test/Testing.md @@ -0,0 +1,33 @@ +TESTING NOTES +============= + +Given the relative imports in the testing files `from ..message import *` +for example, run the tests as so: + +``` +cd TO_PYCOMMAND_ROOT_DIR +python -m irods.test.message_test +``` + +You may also run the tests from within the irods/test/ directory: + +``` +python message_test.py +``` + +Test Dependencies +----------------- + +Testing appears to rely on a local irods directory. + + +| Test | Status | Notes | +| ------------- |-------------|------------| +| browse_test.py | fails | `File "browse_test.py", line 12, in test_get_collection coll = sess.get_collection(path) AttributeError: 'iRODSSession' object has no attribute 'get_collection'`| +| coll_test.py | fails | requires username, password, host; fixable| +| file_test.py | fails | `File "file_test.py", line 9, in obj = sess.get_data_object("/tempZone/home/rods/test1") AttributeError: 'iRODSSession' object has no attribute 'get_data_object'`| +| message_test.py | || +| meta_test.py ||| +| query_test.py ||| +| test_connection.py ||| + diff --git a/irods/test/__init__.py b/irods/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/browse_test.py b/irods/test/browse_test.py old mode 100755 new mode 100644 similarity index 82% rename from browse_test.py rename to irods/test/browse_test.py index 52e9f1c..2f39d95 --- a/browse_test.py +++ b/irods/test/browse_test.py @@ -1,12 +1,17 @@ #!/usr/bin/env python +import os +import sys +if __name__ == '__main__': + sys.path.insert(0, os.path.abspath('../..')) + import unittest from irods.session import iRODSSession -sess = iRODSSession(host='localhost', port=1247, user='rods', password='rods', +sess = iRODSSession(host='localhost', port=1247, user='rods', password='rods', zone='tempZone') class TestMessages(unittest.TestCase): - + def test_get_collection(self): path = "/tempZone/home/rods" coll = sess.get_collection(path) diff --git a/coll_test.py b/irods/test/coll_test.py old mode 100755 new mode 100644 similarity index 76% rename from coll_test.py rename to irods/test/coll_test.py index 8011385..0668b9c --- a/coll_test.py +++ b/irods/test/coll_test.py @@ -1,10 +1,15 @@ #! /usr/bin/env python +import os +import sys +if __name__ == '__main__': + sys.path.insert(0, os.path.abspath('../..')) + from irods.session import iRODSSession from irods.models import Collection, User, DataObject import logging sess = iRODSSession(host='localhost', port=1247, \ - user='rods', password='rods', zone='tempZone') + user='rods', password='rods', zone='tempZone') #q1 = sess.query(Collection.id).filter(Collection.name == "'/tempZone/home/rods'") #q1.all() diff --git a/file_test.py b/irods/test/file_test.py old mode 100755 new mode 100644 similarity index 69% rename from file_test.py rename to irods/test/file_test.py index ecbfcd5..db66ccc --- a/file_test.py +++ b/irods/test/file_test.py @@ -1,10 +1,15 @@ #! /usr/bin/env python +import os +import sys +if __name__ == '__main__': + sys.path.insert(0, os.path.abspath('../..')) + from irods.session import iRODSSession from irods.models import Collection, User, DataObject import logging sess = iRODSSession(host='localhost', port=1247, \ - user='rods', password='rods', zone='tempZone') + user='rods', password='rods', zone='tempZone') obj = sess.get_data_object("/tempZone/home/rods/test1") f = obj.open('w+') diff --git a/message_test.py b/irods/test/message_test.py old mode 100755 new mode 100644 similarity index 97% rename from message_test.py rename to irods/test/message_test.py index 2fec2fd..d0ce840 --- a/message_test.py +++ b/irods/test/message_test.py @@ -1,13 +1,18 @@ #!/usr/bin/env python +import os +import sys +if __name__ == '__main__': + sys.path.insert(0, os.path.abspath('../..')) + import unittest from xml.etree import ElementTree as ET from base64 import b64encode, b64decode -from irods.message import (StartupPack, AuthResponse, IntegerIntegerMap, - IntegerStringMap, StringStringMap, GenQueryRequest, +from irods.message import (StartupPack, AuthResponse, IntegerIntegerMap, + IntegerStringMap, StringStringMap, GenQueryRequest, GenQueryResponseColumn, GenQueryResponse) class TestMessages(unittest.TestCase): - + def test_startup_pack(self): sup = StartupPack() sup.irodsProt = 2 diff --git a/meta_test.py b/irods/test/meta_test.py old mode 100755 new mode 100644 similarity index 77% rename from meta_test.py rename to irods/test/meta_test.py index ccc4d3e..e15c13a --- a/meta_test.py +++ b/irods/test/meta_test.py @@ -1,11 +1,16 @@ #! /usr/bin/env python +import os +import sys +if __name__ == '__main__': + sys.path.insert(0, os.path.abspath('../..')) + import logging from irods.session import iRODSSession from irods.models import Collection, User, DataObject from irods.meta import iRODSMeta sess = iRODSSession(host='localhost', port=1247, \ - user='rods', password='rods', zone='tempZone') + user='rods', password='rods', zone='tempZone') obj = sess.get_data_object("/tempZone/home/rods/test1") diff --git a/query_test.py b/irods/test/query_test.py old mode 100755 new mode 100644 similarity index 81% rename from query_test.py rename to irods/test/query_test.py index 5419210..3be30a4 --- a/query_test.py +++ b/irods/test/query_test.py @@ -1,11 +1,16 @@ #! /usr/bin/env python2.6 +import os +import sys +if __name__ == '__main__': + sys.path.insert(0, os.path.abspath('../..')) + from irods.session import iRODSSession from irods.query import Query from irods.models import User, Collection, Keywords import logging sess = iRODSSession(host='localhost', port=4444, \ - user='rods', password='rods', zone='tempZone') + user='rods', password='rods', zone='tempZone') q1 = sess.query(User, Collection.name) q2 = q1.filter(User.name == 'cjlarose') q3 = q2.filter(Keywords.chksum == '12345') diff --git a/test_connection.py b/irods/test/test_connection.py old mode 100755 new mode 100644 similarity index 50% rename from test_connection.py rename to irods/test/test_connection.py index 5156443..3026ec9 --- a/test_connection.py +++ b/irods/test/test_connection.py @@ -1,6 +1,11 @@ #! /usr/bin/env python2.6 +import os +import sys +if __name__ == '__main__': + sys.path.insert(0, os.path.abspath('../..')) + from irods.session import iRODSSession sess = iRODSSession(host='localhost', port=4444, \ - user='rods', password='rods', zone='tempZone') + user='rods', password='rods', zone='tempZone') coll = sess.get_collection('/tempZone/home/rods') From c0868a96f09bc4167bac9b3e91a29770348d27ab Mon Sep 17 00:00:00 2001 From: Michael Gatto Date: Thu, 29 Aug 2013 16:24:01 -0700 Subject: [PATCH 02/55] Rename to match naming convention of rest of tests in this package. --- irods/test/{test_connection.py => connection_test.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename irods/test/{test_connection.py => connection_test.py} (100%) diff --git a/irods/test/test_connection.py b/irods/test/connection_test.py similarity index 100% rename from irods/test/test_connection.py rename to irods/test/connection_test.py From e37d859ee256066db04b0facb6d29133cd1577d7 Mon Sep 17 00:00:00 2001 From: Michael Gatto Date: Thu, 29 Aug 2013 16:26:11 -0700 Subject: [PATCH 03/55] Add heading for test results. --- irods/test/Testing.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/irods/test/Testing.md b/irods/test/Testing.md index 36530da..6ef8fcc 100644 --- a/irods/test/Testing.md +++ b/irods/test/Testing.md @@ -20,6 +20,9 @@ Test Dependencies Testing appears to rely on a local irods directory. +Current Test Results +-------------------- + | Test | Status | Notes | | ------------- |-------------|------------| From 14f72c43c577066c23b45bf63d3ef0f3db9df4b6 Mon Sep 17 00:00:00 2001 From: Michael Gatto Date: Thu, 29 Aug 2013 16:27:12 -0700 Subject: [PATCH 04/55] Update results. --- irods/test/Testing.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/irods/test/Testing.md b/irods/test/Testing.md index 6ef8fcc..0515205 100644 --- a/irods/test/Testing.md +++ b/irods/test/Testing.md @@ -29,8 +29,8 @@ Current Test Results | browse_test.py | fails | `File "browse_test.py", line 12, in test_get_collection coll = sess.get_collection(path) AttributeError: 'iRODSSession' object has no attribute 'get_collection'`| | coll_test.py | fails | requires username, password, host; fixable| | file_test.py | fails | `File "file_test.py", line 9, in obj = sess.get_data_object("/tempZone/home/rods/test1") AttributeError: 'iRODSSession' object has no attribute 'get_data_object'`| -| message_test.py | || -| meta_test.py ||| -| query_test.py ||| -| test_connection.py ||| +| message_test.py | succeeds || +| meta_test.py | fails || +| query_test.py | fails || +| test_connection.py | fails || From aa2166031bc67255e71d2142c4b3467eadbd0578 Mon Sep 17 00:00:00 2001 From: Michael Gatto Date: Thu, 29 Aug 2013 16:48:22 -0700 Subject: [PATCH 05/55] Remove double import. --- irods/connection.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/irods/connection.py b/irods/connection.py index 2c64049..a09ad2f 100644 --- a/irods/connection.py +++ b/irods/connection.py @@ -2,10 +2,9 @@ import logging import struct import hashlib -import logging -from irods.message import (iRODSMessage, StartupPack, AuthResponse, AuthChallenge, - FileReadRequest, FileWriteRequest, FileSeekRequest, FileSeekResponse, +from irods.message import (iRODSMessage, StartupPack, AuthResponse, AuthChallenge, + FileReadRequest, FileWriteRequest, FileSeekRequest, FileSeekResponse, FileCloseRequest) from irods.exception import get_exception_by_code from irods import MAX_PASSWORD_LENGTH From 7b653cfffa988477556166ef5eeefdbc8047743a Mon Sep 17 00:00:00 2001 From: Chris LaRose Date: Tue, 3 Sep 2013 10:40:20 -0700 Subject: [PATCH 06/55] Update README.md with proxy instructions --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 14ef569..a100f25 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,15 @@ Establishing a connection >>> from irods.session import iRODSSession >>> sess = iRODSSession(host='localhost', port=1247, user='rods', password='rods', zone='tempZone') ``` + +If you're an administrator acting on behalf of another user: +```python +>>> from irods.session import iRODSSession +>>> sess = iRODSSession(host='localhost', port=1247, user='rods', password='rods', zone='tempZone', + client_user='another_user', client_zone='another_zone') +``` + +If no `client_zone` is provided, the `zone` parameter is used in its place. Working with collections ------------------------ From 035f1c62e72787e535e7953037b8f55ef17624d2 Mon Sep 17 00:00:00 2001 From: Chris La Rose Date: Tue, 3 Sep 2013 10:45:01 -0700 Subject: [PATCH 07/55] Removed useless file --- meta.py | 2 -- 1 file changed, 2 deletions(-) delete mode 100755 meta.py diff --git a/meta.py b/meta.py deleted file mode 100755 index c6480ec..0000000 --- a/meta.py +++ /dev/null @@ -1,2 +0,0 @@ -#! /usr/bin/env python2.6 -import irods.models From efea96331d01ea14fc3d20817432f0d96540dbbb Mon Sep 17 00:00:00 2001 From: Chris La Rose Date: Tue, 3 Sep 2013 10:58:13 -0700 Subject: [PATCH 08/55] Added convenience methods for removing data objects and collections. Renamed CollectionManager.[delete=>remove] --- irods/collection.py | 3 +++ irods/data_object.py | 3 +++ irods/resource_manager/collection_manager.py | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/irods/collection.py b/irods/collection.py index e0c67ae..350cdea 100644 --- a/irods/collection.py +++ b/irods/collection.py @@ -37,6 +37,9 @@ def data_objects(self): for row in results ] + def remove(self, recurse=True, force=False, additional_flags={}): + self.manager.remove(self.path, recurse, force, additional_flags) + def walk(self, topdown=True): """ Collection tree generator. diff --git a/irods/data_object.py b/irods/data_object.py index 1b2268f..781bd93 100644 --- a/irods/data_object.py +++ b/irods/data_object.py @@ -36,6 +36,9 @@ def open(self, mode='r'): conn, desc = self.manager.open(self.path, flag) return iRODSDataObjectFile(conn, desc) + def unlink(self): + self.manager.unlink(self.path) + class iRODSDataObjectFile(object): def __init__(self, conn, descriptor): self.conn = conn diff --git a/irods/resource_manager/collection_manager.py b/irods/resource_manager/collection_manager.py index eae1aa7..2f70f6b 100644 --- a/irods/resource_manager/collection_manager.py +++ b/irods/resource_manager/collection_manager.py @@ -26,7 +26,7 @@ def create(self, path): response = conn.recv() return self.get(path) - def delete(self, path, recurse=True, force=False, additional_flags={}): + def remove(self, path, recurse=True, force=False, additional_flags={}): options = {} if recurse: options['recursiveOpr'] = '' From b1ac97d93f209ef9ac3c3035c5a8f0565b4f01ae Mon Sep 17 00:00:00 2001 From: Michael Gatto Date: Tue, 3 Sep 2013 11:13:58 -0700 Subject: [PATCH 09/55] Run all tests at once, if desired. --- irods/test/Testing.md | 9 +++++++++ irods/test/runner.py | 24 ++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 irods/test/runner.py diff --git a/irods/test/Testing.md b/irods/test/Testing.md index 0515205..c4eb821 100644 --- a/irods/test/Testing.md +++ b/irods/test/Testing.md @@ -15,6 +15,15 @@ You may also run the tests from within the irods/test/ directory: python message_test.py ``` +Run All Tests at Once +--------------------- +``` +python runner.py +``` + +This imports all tests in the `test` directory and runs them. It will not die upon any errors. + + Test Dependencies ----------------- diff --git a/irods/test/runner.py b/irods/test/runner.py new file mode 100644 index 0000000..7a3925f --- /dev/null +++ b/irods/test/runner.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +from unittest import TestLoader, TextTestRunner, TestSuite +import sys +import os + +""" +NOTE: "If a test package name (directory with __init__.py) matches the pattern + then the package will be checked for a load_tests function. If this + exists then it will be called with loader, tests, pattern." +""" + +""" +Load all tests in the current directory and run them +""" +if __name__ == "__main__": + # must set the path for the imported tests + sys.path.insert(0, os.path.abspath('../..')) + + loader = TestLoader() + suite = TestSuite(loader.discover(start_dir='.', pattern='*_test.py', + top_level_dir=".")) + + runner = TextTestRunner(verbosity=2) + runner.run(suite) From feb0a4badb5c64b36002b79ed4f6d8b94a721dab Mon Sep 17 00:00:00 2001 From: Chris LaRose Date: Tue, 3 Sep 2013 11:19:41 -0700 Subject: [PATCH 10/55] Updated install link in readme. Updated TODOs --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a100f25..f3ad4c7 100644 --- a/README.md +++ b/README.md @@ -20,14 +20,13 @@ This project should be considered pre-alpha. Here's what works: - [ ] Connection pool management - [ ] Implement gen query result sets as lazy queries - [X] Return empty result sets when CAT_NO_ROWS_FOUND is raised -- [ ] Optimize querying subcollections and data_objects by maintaining a cache -and checking last modified timestamps +- [ ] Manage permissions Installation ------------ pycommands requires Python 2.7. Installation with pip is easy! - pip install git+git://github.com/cjlarose/pycommands.git + pip install git+git://github.com/iPlantCollaborativeOpenSource/pycommands.git Establishing a connection ------------------------- From ae2801d32a646c55fbba91c99ab25a83a2c825dc Mon Sep 17 00:00:00 2001 From: Chris LaRose Date: Tue, 3 Sep 2013 11:33:40 -0700 Subject: [PATCH 11/55] Update TODOs --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index f3ad4c7..ffaf599 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,9 @@ This project should be considered pre-alpha. Here's what works: - [ ] Implement gen query result sets as lazy queries - [X] Return empty result sets when CAT_NO_ROWS_FOUND is raised - [ ] Manage permissions +- [ ] Manage users and groups +- [ ] Manage zones +- [ ] Manage resources Installation ------------ From b70dc8ce4c999526f41dbdf5b77041bf13c18d6b Mon Sep 17 00:00:00 2001 From: Michael Gatto Date: Tue, 3 Sep 2013 14:25:42 -0700 Subject: [PATCH 12/55] Minor change. --- irods/test/runner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/irods/test/runner.py b/irods/test/runner.py index 7a3925f..565ad19 100644 --- a/irods/test/runner.py +++ b/irods/test/runner.py @@ -1,7 +1,9 @@ #!/usr/bin/env python -from unittest import TestLoader, TextTestRunner, TestSuite + import sys import os +from unittest import TestLoader, TextTestRunner, TestSuite + """ NOTE: "If a test package name (directory with __init__.py) matches the pattern From fa5fd8d1f71bb45aab303ec1da325f9b85a5139d Mon Sep 17 00:00:00 2001 From: Michael Gatto Date: Tue, 3 Sep 2013 15:04:56 -0700 Subject: [PATCH 13/55] Major test restructuring. Brings testing into line with accepted norms for unit testing. There is still far to go, since I must determine what the objects under test return so that they may be compared with unittest.assert* methods. --- irods/test/browse_test.py | 23 ++++++++---- irods/test/coll_test.py | 25 ------------- irods/test/collection_test.py | 40 ++++++++++++++++++++ irods/test/config.py | 5 +++ irods/test/connection_test.py | 35 +++++++++++++++--- irods/test/file_test.py | 69 +++++++++++++++++++++++++++-------- irods/test/message_test.py | 28 +++++++------- irods/test/meta_test.py | 64 ++++++++++++++++++++++++-------- irods/test/query_test.py | 60 ++++++++++++++++-------------- 9 files changed, 238 insertions(+), 111 deletions(-) delete mode 100644 irods/test/coll_test.py create mode 100644 irods/test/collection_test.py create mode 100644 irods/test/config.py diff --git a/irods/test/browse_test.py b/irods/test/browse_test.py index 2f39d95..a02b3f6 100644 --- a/irods/test/browse_test.py +++ b/irods/test/browse_test.py @@ -1,24 +1,31 @@ #!/usr/bin/env python import os import sys -if __name__ == '__main__': - sys.path.insert(0, os.path.abspath('../..')) - import unittest -from irods.session import iRODSSession -sess = iRODSSession(host='localhost', port=1247, user='rods', password='rods', - zone='tempZone') class TestMessages(unittest.TestCase): + def setUp(self): + from irods.session import iRODSSession + import config + + self.sess = iRODSSession(host=config.IRODS_SERVER_HOST, + port=config.IRODS_SERVER_PORT, + user=config.IRODS_USER_USERNAME, + password=config.IRODS_USER_PASSWORD, + zone=config.IRODS_SERVER_ZONE) + def test_get_collection(self): path = "/tempZone/home/rods" - coll = sess.get_collection(path) + coll = self.sess.get_collection(path) self.assertEquals(path, coll.path) - new_coll = sess.create_collection("/tempZone/home/rods/test_dir") + new_coll = self.sess.create_collection("/tempZone/home/rods/test_dir") self.assertEquals(new_coll.name, 'test_dir') + if __name__ == "__main__": + # let the tests find the parent irods lib + sys.path.insert(0, os.path.abspath('../..')) unittest.main() diff --git a/irods/test/coll_test.py b/irods/test/coll_test.py deleted file mode 100644 index 0668b9c..0000000 --- a/irods/test/coll_test.py +++ /dev/null @@ -1,25 +0,0 @@ -#! /usr/bin/env python -import os -import sys -if __name__ == '__main__': - sys.path.insert(0, os.path.abspath('../..')) - -from irods.session import iRODSSession -from irods.models import Collection, User, DataObject -import logging - -sess = iRODSSession(host='localhost', port=1247, \ - user='rods', password='rods', zone='tempZone') -#q1 = sess.query(Collection.id).filter(Collection.name == "'/tempZone/home/rods'") -#q1.all() - -#f = open('collquery', 'w') -#f.write(q1._message().pack()) - -#result = sess.query(Collection.id, Collection.owner_name, User.id, User.name)\ -# .filter(Collection.owner_name == "'rods'")\ -# .all() - -result = sess.query(DataObject.id, DataObject.collection_id, DataObject.name, User.name, Collection.name).all() - -print str(result) diff --git a/irods/test/collection_test.py b/irods/test/collection_test.py new file mode 100644 index 0000000..73eab8d --- /dev/null +++ b/irods/test/collection_test.py @@ -0,0 +1,40 @@ +#! /usr/bin/env python +import os +import sys +import unittest + + +class TestCollection(unittest.TestCase): + + def setUp(self): + from irods.session import iRODSSession + import config + + self.sess = iRODSSession(host=config.IRODS_SERVER_HOST, + port=config.IRODS_SERVER_PORT, + user=config.IRODS_USER_USERNAME, + password=config.IRODS_USER_PASSWORD, + zone=config.IRODS_SERVER_ZONE) + + def test_collection(self): + from irods.models import Collection, User, DataObject + + #q1 = sess.query(Collection.id).filter(Collection.name == "'/tempZone/home/rods'") + #q1.all() + + #f = open('collquery', 'w') + #f.write(q1._message().pack()) + + #result = sess.query(Collection.id, Collection.owner_name, User.id, User.name)\ + # .filter(Collection.owner_name == "'rods'")\ + # .all() + + result = self.sess.query(DataObject.id, DataObject.collection_id, DataObject.name, User.name, Collection.name).all() + + print str(result) + + +if __name__ == "__main__": + # let the tests find the parent irods lib + sys.path.insert(0, os.path.abspath('../..')) + unittest.main() diff --git a/irods/test/config.py b/irods/test/config.py new file mode 100644 index 0000000..6e5544f --- /dev/null +++ b/irods/test/config.py @@ -0,0 +1,5 @@ +IRODS_SERVER_HOST = "localhost" +IRODS_SERVER_PORT = "1247" +IRODS_SERVER_ZONE = "tempZone" +IRODS_USER_USERNAME = "rods" +IRODS_USER_PASSWORD = "rods" diff --git a/irods/test/connection_test.py b/irods/test/connection_test.py index 3026ec9..6136f5a 100644 --- a/irods/test/connection_test.py +++ b/irods/test/connection_test.py @@ -1,11 +1,34 @@ #! /usr/bin/env python2.6 +import unittest import os import sys -if __name__ == '__main__': - sys.path.insert(0, os.path.abspath('../..')) -from irods.session import iRODSSession -sess = iRODSSession(host='localhost', port=4444, \ - user='rods', password='rods', zone='tempZone') -coll = sess.get_collection('/tempZone/home/rods') +class TestConnections(unittest.TestCase): + """ + """ + + def setUp(self): + from irods.session import iRODSSession + import config + + self.sess = iRODSSession(host=config.IRODS_SERVER_HOST, + port=config.IRODS_SERVER_PORT, # 4444: why? + user=config.IRODS_USER_USERNAME, + password=config.IRODS_USER_PASSWORD, + zone=config.IRODS_SERVER_ZONE) + + def test_connection(self): + """ + @TODO: what does get_collection return? + There should be a better way to test this... + Wouldn't the iRODSSession init establish the connection? + """ + coll = self.sess.get_collection('/tempZone/home/rods') + self.assertTrue(coll, "Connection failed.") + + +if __name__ == '__main__': + # let the tests find the parent irods lib + sys.path.insert(0, os.path.abspath('../..')) + unittest.main() diff --git a/irods/test/file_test.py b/irods/test/file_test.py index db66ccc..23a428c 100644 --- a/irods/test/file_test.py +++ b/irods/test/file_test.py @@ -1,27 +1,64 @@ #! /usr/bin/env python +import unittest import os import sys + + +class TestFiles(unittest.TestCase): + """ + """ + + def setUp(self): + from irods.session import iRODSSession + import config + + self.sess = iRODSSession(host=config.IRODS_SERVER_HOST, + port=config.IRODS_SERVER_PORT, # 4444: why? + user=config.IRODS_USER_USERNAME, + password=config.IRODS_USER_PASSWORD, + zone=config.IRODS_SERVER_ZONE) + + def test_file_open(self): + #from irods.models import Collection, User, DataObject + + obj = self.sess.get_data_object("/tempZone/home/rods/test1") + f = obj.open('r+') + f.seek(0, 0) # what does this return? + str1 = f.read() + #self.assertTrue(expr, msg) + f.close() + + def test_file_read(self): + #from irods.models import Collection, User, DataObject + + obj = self.sess.get_data_object("/tempZone/home/rods/test1") + f = obj.open('r+') + str1 = f.read(1024) + #self.assertTrue(expr, msg) + f.close() + + def test_file_write(self): + #from irods.models import Collection, User, DataObject + + obj = self.sess.get_data_object("/tempZone/home/rods/test1") + f = obj.open('w+') + f.write("NEW STRING.py") + f.seek(-6, 2) + f.write("INTERRUPT") + #self.assertTrue(expr, msg) + f.close() + + if __name__ == '__main__': + # let the tests find the parent irods lib sys.path.insert(0, os.path.abspath('../..')) + unittest.main() + + + -from irods.session import iRODSSession -from irods.models import Collection, User, DataObject -import logging -sess = iRODSSession(host='localhost', port=1247, \ - user='rods', password='rods', zone='tempZone') -obj = sess.get_data_object("/tempZone/home/rods/test1") -f = obj.open('w+') -str = f.read(1024) -logging.debug(str) -f.write("NEW STRING.py") -f.seek(-6, 2) -f.write("INTERRUPT") -f.seek(0, 0) -str = f.read() -logging.debug(str) -f.close() diff --git a/irods/test/message_test.py b/irods/test/message_test.py index d0ce840..020ebfb 100644 --- a/irods/test/message_test.py +++ b/irods/test/message_test.py @@ -1,18 +1,20 @@ #!/usr/bin/env python import os import sys + +# this does not get called when imported from runner.py if __name__ == '__main__': sys.path.insert(0, os.path.abspath('../..')) import unittest from xml.etree import ElementTree as ET -from base64 import b64encode, b64decode +#from base64 import b64encode, b64decode from irods.message import (StartupPack, AuthResponse, IntegerIntegerMap, IntegerStringMap, StringStringMap, GenQueryRequest, GenQueryResponseColumn, GenQueryResponse) -class TestMessages(unittest.TestCase): +class TestMessages(unittest.TestCase): def test_startup_pack(self): sup = StartupPack() sup.irodsProt = 2 @@ -68,8 +70,8 @@ def test_auth_response(self): def test_inx_ival_pair(self): iip = IntegerIntegerMap() iip.iiLen = 2 - iip.inx = [4,5] - iip.ivalue = [1,2] + iip.inx = [4, 5] + iip.ivalue = [1, 2] expected = "\ 2\ 4\ @@ -82,8 +84,8 @@ def test_inx_ival_pair(self): iip2 = IntegerIntegerMap() iip2.unpack(ET.fromstring(expected)) self.assertEqual(iip2.iiLen, 2) - self.assertEqual(iip2.inx, [4,5]) - self.assertEqual(iip2.ivalue, [1,2]) + self.assertEqual(iip2.inx, [4, 5]) + self.assertEqual(iip2.ivalue, [1, 2]) def test_key_val_pair(self): kvp = StringStringMap() @@ -119,12 +121,12 @@ def test_gen_query_inp(self): iip = IntegerIntegerMap() iip.iiLen = 2 - iip.inx = [4,5] - iip.ivalue = [1,2] + iip.inx = [4, 5] + iip.ivalue = [1, 2] ivp = IntegerStringMap() ivp.isLen = 2 - ivp.inx = [1,2] + ivp.inx = [1, 2] ivp.svalue = ['five', 'six'] gq.KeyValPair_PI = kvp @@ -146,12 +148,12 @@ def test_gen_query_inp(self): self.assertEqual(gq2.KeyValPair_PI.svalue, ["three", "four"]) self.assertEqual(gq2.InxIvalPair_PI.iiLen, 2) - self.assertEqual(gq2.InxIvalPair_PI.inx, [4,5]) - self.assertEqual(gq2.InxIvalPair_PI.ivalue, [1,2]) + self.assertEqual(gq2.InxIvalPair_PI.inx, [4, 5]) + self.assertEqual(gq2.InxIvalPair_PI.ivalue, [1, 2]) self.assertEqual(gq2.InxValPair_PI.isLen, 2) - self.assertEqual(gq2.InxValPair_PI.inx, [1,2]) - self.assertEqual(gq2.InxValPair_PI.svalue, ["five","six"]) + self.assertEqual(gq2.InxValPair_PI.inx, [1, 2]) + self.assertEqual(gq2.InxValPair_PI.svalue, ["five", "six"]) self.assertEqual(gq2.pack(), expected) diff --git a/irods/test/meta_test.py b/irods/test/meta_test.py index e15c13a..a96f034 100644 --- a/irods/test/meta_test.py +++ b/irods/test/meta_test.py @@ -1,26 +1,58 @@ #! /usr/bin/env python +import unittest import os import sys -if __name__ == '__main__': - sys.path.insert(0, os.path.abspath('../..')) -import logging -from irods.session import iRODSSession -from irods.models import Collection, User, DataObject -from irods.meta import iRODSMeta -sess = iRODSSession(host='localhost', port=1247, \ - user='rods', password='rods', zone='tempZone') +class TestMeta(unittest.TestCase): + """ + """ + + def setUp(self): + from irods.session import iRODSSession + import config + + self.sess = iRODSSession(host=config.IRODS_SERVER_HOST, + port=config.IRODS_SERVER_PORT, + user=config.IRODS_USER_USERNAME, + password=config.IRODS_USER_PASSWORD, + zone=config.IRODS_SERVER_ZONE) + + def test_get_meta(self): + """ + """ + #from irods.meta import iRODSMeta -obj = sess.get_data_object("/tempZone/home/rods/test1") + #obj = self.sess.get_data_object("/tempZone/home/rods/test1") + meta = self.sess.get_meta('d', "/tempZone/home/rods/test1") + print meta + #self.assertEqual(first, second, msg) -meta = sess.get_meta('d', "/tempZone/home/rods/test1") -print meta + def test_add_meta(self): + """ + """ + from irods.meta import iRODSMeta -sess.add_meta('d', '/tempZone/home/rods/test1', iRODSMeta('key8', 'value5')) -sess.remove_meta('d', '/tempZone/home/rods/test1', iRODSMeta('key8', 'value5')) + self.sess.add_meta('d', '/tempZone/home/rods/test1', + iRODSMeta('key8', 'value5')) -sess.copy_meta('d', 'd', '/tempZone/home/rods/test1', '/tempZone/home/rods/test2') + def test_copy_meta(self): + """ + """ + #from irods.meta import iRODSMeta + self.sess.copy_meta('d', 'd', '/tempZone/home/rods/test1', + '/tempZone/home/rods/test2') -#meta = sess.get_meta('d', "/tempZone/home/rods/test1") -#print meta + def test_remove_meta(self): + """ + """ + from irods.meta import iRODSMeta + + self.sess.remove_meta('d', '/tempZone/home/rods/test1', + iRODSMeta('key8', 'value5')) + + +if __name__ == '__main__': + # let the tests find the parent irods lib + sys.path.insert(0, os.path.abspath('../..')) + unittest.main() diff --git a/irods/test/query_test.py b/irods/test/query_test.py index 3be30a4..973ec96 100644 --- a/irods/test/query_test.py +++ b/irods/test/query_test.py @@ -1,41 +1,47 @@ #! /usr/bin/env python2.6 +import unittest import os import sys -if __name__ == '__main__': - sys.path.insert(0, os.path.abspath('../..')) -from irods.session import iRODSSession -from irods.query import Query -from irods.models import User, Collection, Keywords -import logging -sess = iRODSSession(host='localhost', port=4444, \ - user='rods', password='rods', zone='tempZone') -q1 = sess.query(User, Collection.name) -q2 = q1.filter(User.name == 'cjlarose') -q3 = q2.filter(Keywords.chksum == '12345') +class TestQuery(unittest.TestCase): + """ + """ + + def setUp(self): + from irods.session import iRODSSession + import config -logging.debug(q1.columns) -logging.debug(q1.criteria) + self.sess = iRODSSession(host=config.IRODS_SERVER_HOST, + port=config.IRODS_SERVER_PORT, # 4444 why? + user=config.IRODS_USER_USERNAME, + password=config.IRODS_USER_PASSWORD, + zone=config.IRODS_SERVER_ZONE) -logging.debug(q2.columns) -logging.debug(q2.criteria) + def test_query(self): + #from irods.query import Query + from irods.models import User, Collection, Keywords -logging.debug(q3.columns) -logging.debug(q3.criteria) + q1 = self.sess.query(User, Collection.name) + q2 = q1.filter(User.name == 'cjlarose') + q3 = q2.filter(Keywords.chksum == '12345') -logging.debug(q3._select_message().pack()) + f = open('select', 'w') + f.write(q3._select_message().pack()) -f = open('select', 'w') -f.write(q3._select_message().pack()) + f = open('conds', 'w') + f.write(q3._conds_message().pack()) -f = open('conds', 'w') -f.write(q3._conds_message().pack()) + f = open('condskw', 'w') + f.write(q3._kw_message().pack()) -f = open('condskw', 'w') -f.write(q3._kw_message().pack()) + f = open('genq', 'w') + f.write(q3._message().pack()) -f = open('genq', 'w') -f.write(q3._message().pack()) + self.sess.query(Collection.id, Collection.name).all() -sess.query(Collection.id, Collection.name).all() + +if __name__ == '__main__': + # let the tests find the parent irods lib + sys.path.insert(0, os.path.abspath('../..')) + unittest.main() From f98fdceb070c9a4a83c35da09280244c3d8e8a65 Mon Sep 17 00:00:00 2001 From: Chris La Rose Date: Thu, 5 Sep 2013 11:02:12 -0700 Subject: [PATCH 14/55] Added license --- LICENSE.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 LICENSE.txt diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..f970097 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,13 @@ +Copyright (c) 2010-2013, The Arizona Board of Regents on behalf of The University of Arizona + +All rights reserved. + +Developed by: iPlant Collaborative as a collaboration between participants at BIO5 at The University of Arizona (the primary hosting institution), Cold Spring Harbor Laboratory, The University of Texas at Austin, and individual contributors. Find out more at http://www.iplantcollaborative.org/. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * Neither the name of the iPlant Collaborative, BIO5, The University of Arizona, Cold Spring Harbor Laboratory, The University of Texas at Austin, nor the names of other contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 62ba296b2d3f5ab35c159d061461654d5906287a Mon Sep 17 00:00:00 2001 From: Michael Gatto Date: Fri, 6 Sep 2013 10:58:00 -0700 Subject: [PATCH 15/55] Consolidate tests. Add a bunch of skippable tests since feature may be as yet unimplemented. Rationalize location of some tests, especially 'browse_test.py' since its test is really a type of collection test. --- irods/test/browse_test.py | 31 ---------------------------- irods/test/collection_test.py | 39 +++++++++++++++++++++++++---------- irods/test/connection_test.py | 6 ++++++ irods/test/query_test.py | 19 +++++++++++++++++ 4 files changed, 53 insertions(+), 42 deletions(-) delete mode 100644 irods/test/browse_test.py diff --git a/irods/test/browse_test.py b/irods/test/browse_test.py deleted file mode 100644 index a02b3f6..0000000 --- a/irods/test/browse_test.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import unittest - - -class TestMessages(unittest.TestCase): - - def setUp(self): - from irods.session import iRODSSession - import config - - self.sess = iRODSSession(host=config.IRODS_SERVER_HOST, - port=config.IRODS_SERVER_PORT, - user=config.IRODS_USER_USERNAME, - password=config.IRODS_USER_PASSWORD, - zone=config.IRODS_SERVER_ZONE) - - def test_get_collection(self): - path = "/tempZone/home/rods" - coll = self.sess.get_collection(path) - self.assertEquals(path, coll.path) - - new_coll = self.sess.create_collection("/tempZone/home/rods/test_dir") - self.assertEquals(new_coll.name, 'test_dir') - - -if __name__ == "__main__": - # let the tests find the parent irods lib - sys.path.insert(0, os.path.abspath('../..')) - unittest.main() diff --git a/irods/test/collection_test.py b/irods/test/collection_test.py index 73eab8d..7072b20 100644 --- a/irods/test/collection_test.py +++ b/irods/test/collection_test.py @@ -5,6 +5,7 @@ class TestCollection(unittest.TestCase): + test_coll_path = "/tempZone/home/rods/test_dir" def setUp(self): from irods.session import iRODSSession @@ -16,23 +17,39 @@ def setUp(self): password=config.IRODS_USER_PASSWORD, zone=config.IRODS_SERVER_ZONE) - def test_collection(self): - from irods.models import Collection, User, DataObject + self.coll = self.sess.create_collection(self.test_coll_path) - #q1 = sess.query(Collection.id).filter(Collection.name == "'/tempZone/home/rods'") - #q1.all() + def tearDown(self): + """ Delete the test collection after each test """ + self.coll.remove() - #f = open('collquery', 'w') - #f.write(q1._message().pack()) + def test_get_collection(self): + #path = "/tempZone/home/rods" + coll = self.sess.get_collection(self.test_coll_path) + self.assertEquals(self.test_coll_path, coll.path) - #result = sess.query(Collection.id, Collection.owner_name, User.id, User.name)\ - # .filter(Collection.owner_name == "'rods'")\ - # .all() + #def test_new_collection(self): + # self.assertEquals(self.coll.name, 'test_dir') - result = self.sess.query(DataObject.id, DataObject.collection_id, DataObject.name, User.name, Collection.name).all() + def test_append_to_collection(self): + """ Append a new file to the collection""" + pass - print str(result) + def test_remove_from_collection(self): + """ Delete a file from a collection """ + pass + def test_update_in_collection(self): + """ Modify a file in a collection """ + pass + + def test_move_collection(self): + new_path = "/tempZone/home/rods/test_dir_moved" + self.coll.move(new_path) + self.assertEquals(new_path, self.coll.path) + + #def test_delete_collection(self): + # pass if __name__ == "__main__": # let the tests find the parent irods lib diff --git a/irods/test/connection_test.py b/irods/test/connection_test.py index 6136f5a..952cd81 100644 --- a/irods/test/connection_test.py +++ b/irods/test/connection_test.py @@ -27,6 +27,12 @@ def test_connection(self): coll = self.sess.get_collection('/tempZone/home/rods') self.assertTrue(coll, "Connection failed.") + @unittest.skip("unimplemented") + def test_failed_connection(self): + """ Test the exception raised by a failed connection """ + #self.assertRaises() How to fuddle the config.* to ensure setUp() + # fails in connecting? + pass if __name__ == '__main__': # let the tests find the parent irods lib diff --git a/irods/test/query_test.py b/irods/test/query_test.py index 973ec96..559b0e8 100644 --- a/irods/test/query_test.py +++ b/irods/test/query_test.py @@ -40,6 +40,25 @@ def test_query(self): self.sess.query(Collection.id, Collection.name).all() + """ + cut-n-pasted from collection_test... + """ + from irods.models import Collection, User, DataObject + + #q1 = sess.query(Collection.id).filter(Collection.name == "'/tempZone/home/rods'") + #q1.all() + + #f = open('collquery', 'w') + #f.write(q1._message().pack()) + + #result = sess.query(Collection.id, Collection.owner_name, User.id, User.name)\ + # .filter(Collection.owner_name == "'rods'")\ + # .all() + + result = self.sess.query(DataObject.id, DataObject.collection_id, DataObject.name, User.name, Collection.name).all() + + print str(result) + if __name__ == '__main__': # let the tests find the parent irods lib From cc449577a88c56a409d7ab29fbbd2f7bf5706146 Mon Sep 17 00:00:00 2001 From: Steve Gregory Date: Mon, 16 Sep 2013 14:50:44 -0700 Subject: [PATCH 16/55] Updated version to 0.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 740f047..9bb3c21 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from distutils.core import setup setup( name='pycommands', - version='0.0', + version='0.1', author='Chris LaRose', author_email='cjlarose@iplantcollaborative.org', packages=['irods', 'irods.message', 'irods.resource_manager'], From 547abb3f2752206966f05fa1db60a84ed406acb5 Mon Sep 17 00:00:00 2001 From: JMatt Peterson Date: Tue, 24 Sep 2013 14:53:13 -0700 Subject: [PATCH 17/55] Rename from pycommands to python-irodsclient --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ffaf599..1e143a8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -pycommands +python-irodsclient ============ [iRODS](https://www.irods.org) is an open-source distributed filesystem manager. This a client API implemented in python. From 9a3d89e24aef90fa51d76fd6c34d033fc471e9bd Mon Sep 17 00:00:00 2001 From: Chris LaRose Date: Mon, 14 Oct 2013 20:33:16 -0700 Subject: [PATCH 18/55] Changed project name in setup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9bb3c21..23c9b60 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from distutils.core import setup setup( - name='pycommands', + name='python-irodsclient', version='0.1', author='Chris LaRose', author_email='cjlarose@iplantcollaborative.org', From e071f3dd4f1541a07441bea6166eb2256808a3d4 Mon Sep 17 00:00:00 2001 From: Chris LaRose Date: Mon, 14 Oct 2013 21:48:45 -0700 Subject: [PATCH 19/55] Implmented iRODSDataObjectFile.[readline(), readlines()] --- irods/data_object.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/irods/data_object.py b/irods/data_object.py index 781bd93..17581f1 100644 --- a/irods/data_object.py +++ b/irods/data_object.py @@ -44,6 +44,7 @@ def __init__(self, conn, descriptor): self.conn = conn self.desc = descriptor self.position = 0 + self.buffer = "" def tell(self): return self.position @@ -97,11 +98,37 @@ def __iter__(self): else: chars.append(char) + # This implementation is very naive. Refactor with io module + # An empty buffer indicates we haven't filled the buffer, whereas a null + # buffer indicates that we've reached EOF def readline(self): - pass + + if self.buffer == None: + return "" + + while True: + nl = self.buffer.find('\n') + if nl == -1: + contents = self.read(4096) + if contents: + self.buffer += contents + else: #EOF + line = self.buffer + self.buffer = None + return line + else: + line = self.buffer[:(nl+1)] + self.buffer = self.buffer[(nl+1):] + return line def readlines(self): - pass + lines = [] + while True: + line = self.readline() + if not line: + break + lines.append(line) + return lines def __enter__(self): return self From 77e6cb80294da2e7d31d5c4b56fece09b24fd3a1 Mon Sep 17 00:00:00 2001 From: Falmarri Date: Wed, 16 Oct 2013 16:02:46 -0700 Subject: [PATCH 20/55] readlines is a generator --- irods/data_object.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/irods/data_object.py b/irods/data_object.py index 17581f1..241800b 100644 --- a/irods/data_object.py +++ b/irods/data_object.py @@ -122,13 +122,11 @@ def readline(self): return line def readlines(self): - lines = [] while True: line = self.readline() if not line: break - lines.append(line) - return lines + yield line def __enter__(self): return self From d08e7045efabe3bb79f0534f78ca2dcf2ccc77dd Mon Sep 17 00:00:00 2001 From: Christopher La Rose Date: Fri, 18 Oct 2013 13:30:15 -0700 Subject: [PATCH 21/55] Fixed null comparison --- irods/data_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/irods/data_object.py b/irods/data_object.py index 241800b..7ba8a79 100644 --- a/irods/data_object.py +++ b/irods/data_object.py @@ -103,7 +103,7 @@ def __iter__(self): # buffer indicates that we've reached EOF def readline(self): - if self.buffer == None: + if self.buffer is None: return "" while True: From 96aeaee2baaa74f0ff183e63c98edb10e56710a4 Mon Sep 17 00:00:00 2001 From: Chris La Rose Date: Fri, 20 Dec 2013 16:43:35 -0700 Subject: [PATCH 22/55] Reimplemented buffered reading and writing with new io module --- irods/data_object.py | 96 +++++++++----------------------------------- 1 file changed, 20 insertions(+), 76 deletions(-) diff --git a/irods/data_object.py b/irods/data_object.py index 7ba8a79..070e384 100644 --- a/irods/data_object.py +++ b/irods/data_object.py @@ -1,4 +1,5 @@ from os import O_RDONLY, O_WRONLY, O_RDWR +from io import RawIOBase, BufferedRandom from irods.models import DataObject from irods.meta import iRODSMetaCollection @@ -33,21 +34,17 @@ def open(self, mode='r'): 'a': (O_WRONLY, True, True), 'a+': (O_RDWR, True, True), }[mode] + # TODO: Actually use create_if_not_exists and seek_to_end conn, desc = self.manager.open(self.path, flag) - return iRODSDataObjectFile(conn, desc) + return BufferedRandom(iRODSDataObjectFileRaw(conn, desc)) def unlink(self): self.manager.unlink(self.path) -class iRODSDataObjectFile(object): +class iRODSDataObjectFileRaw(RawIOBase): def __init__(self, conn, descriptor): self.conn = conn self.desc = descriptor - self.position = 0 - self.buffer = "" - - def tell(self): - return self.position def close(self): try: @@ -58,78 +55,25 @@ def close(self): self.conn.release() return None - def read(self, size=None): - if not size: - return "".join(self.read_gen()()) - contents = self.conn.read_file(self.desc, size) - if contents: - self.position += len(contents) - return contents - - def read_gen(self, chunk_size=4096, close=False): - def make_gen(): - while True: - contents = self.read(chunk_size) - if not contents: - break - yield contents - if close: - self.close() - return make_gen - - def write(self, string): - written = self.conn.write_file(self.desc, string) - self.position += written - return None - def seek(self, offset, whence=0): - pos = self.conn.seek_file(self.desc, offset, whence) - self.position = pos - pass - - def __iter__(self): - reader = self.read_gen() - chars = [] - for chunk in reader(): - for char in chunk: - if char == '\n': - yield "".join(chars) - chars = [] - else: - chars.append(char) - - # This implementation is very naive. Refactor with io module - # An empty buffer indicates we haven't filled the buffer, whereas a null - # buffer indicates that we've reached EOF - def readline(self): + return self.conn.seek_file(self.desc, offset, whence) - if self.buffer is None: - return "" + def readinto(self, b): + contents = self.conn.read_file(self.desc, len(b)) + if contents is None: + return 0 + for i, c in enumerate(contents): + b[i] = c + return len(contents) - while True: - nl = self.buffer.find('\n') - if nl == -1: - contents = self.read(4096) - if contents: - self.buffer += contents - else: #EOF - line = self.buffer - self.buffer = None - return line - else: - line = self.buffer[:(nl+1)] - self.buffer = self.buffer[(nl+1):] - return line + def write(self, b): + return self.conn.write_file(self.desc, str(b.tobytes())) - def readlines(self): - while True: - line = self.readline() - if not line: - break - yield line + def readable(self): + return True - def __enter__(self): - return self + def writable(self): + return True - def __exit__(self, exc_type, exc_value, traceback): - self.close() + def seekable(self): + return True From fc49d5aaded624f55a15703711c37e89b57745fa Mon Sep 17 00:00:00 2001 From: Chris La Rose Date: Fri, 20 Dec 2013 17:35:13 -0700 Subject: [PATCH 23/55] Fixed closing files --- irods/data_object.py | 1 + 1 file changed, 1 insertion(+) diff --git a/irods/data_object.py b/irods/data_object.py index 070e384..fdb86b1 100644 --- a/irods/data_object.py +++ b/irods/data_object.py @@ -53,6 +53,7 @@ def close(self): pass finally: self.conn.release() + super(iRODSDataObjectFileRaw, self).close() return None def seek(self, offset, whence=0): From 75e302da317690e410e4abe3cef4ee06dd606f38 Mon Sep 17 00:00:00 2001 From: "J. Matt Peterson" Date: Wed, 1 Jan 2014 23:55:09 -0700 Subject: [PATCH 24/55] Fix bug where port keyword didn't work if it was a string. This bug was breaking tests and silently failed when creating irods sessions. modified: session.py --- irods/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/irods/session.py b/irods/session.py index e556524..878452e 100644 --- a/irods/session.py +++ b/irods/session.py @@ -18,7 +18,7 @@ def __init__(self, *args, **kwargs): def configure(self, host=None, port=1247, user=None, zone=None, password=None, client_user=None, client_zone=None): - account = iRODSAccount(host, port, user, zone, password, client_user, + account = iRODSAccount(host, int(port), user, zone, password, client_user, client_zone) self.pool = Pool(account) From d88b45beb1b969870e149a9d822d6d194da21ed6 Mon Sep 17 00:00:00 2001 From: Christopher La Rose Date: Tue, 7 Jan 2014 11:58:33 -0700 Subject: [PATCH 25/55] Replaced instances of logging.{debug, info, warn, error} with logging.getLogger(__name__).{debug, info, warn, error} for better logging support --- irods/connection.py | 8 +++++--- irods/message/__init__.py | 9 ++++----- irods/models.py | 6 ------ irods/pool.py | 6 ++++-- irods/query.py | 1 - irods/resource_manager/metadata_manager.py | 8 +++++--- irods/results.py | 1 - irods/session.py | 2 -- 8 files changed, 18 insertions(+), 23 deletions(-) diff --git a/irods/connection.py b/irods/connection.py index 0f08ca9..92255a0 100644 --- a/irods/connection.py +++ b/irods/connection.py @@ -10,6 +10,8 @@ from irods import MAX_PASSWORD_LENGTH from irods.api_number import api_number +logger = logging.getLogger(__name__) + class Connection(object): def __init__(self, pool, account): self.pool = pool @@ -24,7 +26,7 @@ def __del__(self): def send(self, message): str = message.pack() - logging.debug(str) + logger.debug(str) return self.socket.sendall(str) def recv(self): @@ -72,7 +74,7 @@ def _login(self): # challenge challenge_msg = self.recv() - logging.debug(challenge_msg.msg) + logger.debug(challenge_msg.msg) challenge = challenge_msg.get_main_message(AuthChallenge).challenge padded_pwd = struct.pack("%ds" % MAX_PASSWORD_LENGTH, self.account.password) m = hashlib.md5() @@ -95,7 +97,7 @@ def read_file(self, desc, size): message = iRODSMessage('RODS_API_REQ', msg=message_body, int_info=api_number['DATA_OBJ_READ201_AN']) - logging.debug(desc) + logger.debug(desc) self.send(message) response = self.recv() return response.bs diff --git a/irods/message/__init__.py b/irods/message/__init__.py index 5f7ff06..9484f2d 100644 --- a/irods/message/__init__.py +++ b/irods/message/__init__.py @@ -8,6 +8,8 @@ IntegerProperty, LongProperty, ArrayProperty, SubmessageProperty) +logger = logging.getLogger(__name__) + class iRODSMessage(object): def __init__(self, type=None, msg=None, error=None, bs=None, int_info=None): self.type = type @@ -19,10 +21,8 @@ def __init__(self, type=None, msg=None, error=None, bs=None, int_info=None): @staticmethod def recv(sock): rsp_header_size = sock.recv(4, socket.MSG_WAITALL) - logging.debug(rsp_header_size) rsp_header_size = struct.unpack(">i", rsp_header_size)[0] rsp_header = sock.recv(rsp_header_size, socket.MSG_WAITALL) - logging.debug(rsp_header) xml_root = ET.fromstring(rsp_header) type = xml_root.find('type').text @@ -36,7 +36,7 @@ def recv(sock): bs = sock.recv(bs_len, socket.MSG_WAITALL) if bs_len != 0 else None #if message: - #logging.debug(message) + #logger.debug(message) return iRODSMessage(type, message, error, bs, int_info) @@ -58,8 +58,7 @@ def pack(self): def get_main_message(self, cls): msg = cls() - logging.debug(self.msg) - logging.debug(len(self.msg)) + logger.debug(self.msg) msg.unpack(ET.fromstring(self.msg)) return msg diff --git a/irods/models.py b/irods/models.py index 8f9ce93..7c7bb4b 100644 --- a/irods/models.py +++ b/irods/models.py @@ -1,19 +1,13 @@ -import logging - from irods.column import Column, Integer, String, DateTime, Keyword class ModelBase(type): columns = {} def __new__(cls, name, bases, attr): - #logging.debug(name) - #logging.debug(attr.iteritems()) columns = [y for (x,y) in attr.iteritems() if isinstance(y, Column)] for col in columns: ModelBase.columns[col.icat_id] = col - #logging.debug(columns) attr['_columns'] = columns #attr['_icat_column_names'] = [y.icat_key for (x,y) in columns] - #logging.debug(attr['_icat_column_names']) return type.__new__(cls, name, bases, attr) class Model(object): diff --git a/irods/pool.py b/irods/pool.py index 94c6b99..114aec5 100644 --- a/irods/pool.py +++ b/irods/pool.py @@ -2,6 +2,8 @@ from irods.connection import Connection +logger = logging.getLogger(__name__) + class Pool(object): def __init__(self, account): self.account = account @@ -15,10 +17,10 @@ def get_connection(self): except KeyError: conn = Connection(self, self.account) self.active.add(conn) - logging.debug('num active: %d' % len(self.active)) + logger.debug('num active: %d' % len(self.active)) return conn def release_connection(self, conn): self.active.remove(conn) self.idle.add(conn) - logging.debug('num idle: %d' % len(self.idle)) + logger.debug('num idle: %d' % len(self.idle)) diff --git a/irods/query.py b/irods/query.py index 349f309..d6c2a97 100644 --- a/irods/query.py +++ b/irods/query.py @@ -1,4 +1,3 @@ -import logging from collections import OrderedDict from irods.models import Model diff --git a/irods/resource_manager/metadata_manager.py b/irods/resource_manager/metadata_manager.py index 70edd01..18938cd 100644 --- a/irods/resource_manager/metadata_manager.py +++ b/irods/resource_manager/metadata_manager.py @@ -8,6 +8,8 @@ CollectionMeta, ResourceMeta, UserMeta) from irods.meta import iRODSMeta +logger = logging.getLogger(__name__) + class MetadataManager(ResourceManager): @staticmethod def _model_class_to_resource_type(model_cls): @@ -59,7 +61,7 @@ def add(self, model_cls, path, meta): with self.sess.pool.get_connection() as conn: conn.send(request) response = conn.recv() - logging.debug(response.int_info) + logger.debug(response.int_info) def remove(self, model_cls, path, meta): resource_type = self._model_class_to_resource_type(model_cls) @@ -76,7 +78,7 @@ def remove(self, model_cls, path, meta): with self.sess.pool.get_connection() as conn: conn.send(request) response = conn.recv() - logging.debug(response.int_info) + logger.debug(response.int_info) def copy(self, src_model_cls, dest_model_cls, src, dest): src_resource_type = self._model_class_to_resource_type(src_model_cls) @@ -94,4 +96,4 @@ def copy(self, src_model_cls, dest_model_cls, src, dest): with self.sess.pool.get_connection() as conn: conn.send(request) response = conn.recv() - logging.debug(response.int_info) + logger.debug(response.int_info) diff --git a/irods/results.py b/irods/results.py index a89067b..7b01b72 100644 --- a/irods/results.py +++ b/irods/results.py @@ -1,4 +1,3 @@ -import logging from prettytable import PrettyTable from irods.models import ModelBase diff --git a/irods/session.py b/irods/session.py index 878452e..ebfd5d9 100644 --- a/irods/session.py +++ b/irods/session.py @@ -1,5 +1,3 @@ -import logging - from irods.query import Query from irods.pool import Pool from irods.account import iRODSAccount From ae1021d629fae8579f52ade0a364bb9949738c06 Mon Sep 17 00:00:00 2001 From: Falmarri Date: Thu, 9 Jan 2014 14:19:48 -0700 Subject: [PATCH 26/55] Thread safe connection pool --- irods/pool.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/irods/pool.py b/irods/pool.py index 114aec5..d2bfcad 100644 --- a/irods/pool.py +++ b/irods/pool.py @@ -1,4 +1,5 @@ import logging +import threading from irods.connection import Connection @@ -7,20 +8,22 @@ class Pool(object): def __init__(self, account): self.account = account - + self._lock = threading.Lock() self.active = set() self.idle = set() def get_connection(self): - try: - conn = self.idle.pop() - except KeyError: - conn = Connection(self, self.account) - self.active.add(conn) + with self._lock: + try: + conn = self.idle.pop() + except KeyError: + conn = Connection(self, self.account) + self.active.add(conn) logger.debug('num active: %d' % len(self.active)) return conn def release_connection(self, conn): - self.active.remove(conn) - self.idle.add(conn) + with self._lock: + self.active.remove(conn) + self.idle.add(conn) logger.debug('num idle: %d' % len(self.idle)) From 5804bbe1b12189d439e621e3f1c7d190c2f22b1e Mon Sep 17 00:00:00 2001 From: Chris La Rose Date: Mon, 13 Jan 2014 00:35:57 -0700 Subject: [PATCH 27/55] Corrected typo in exception --- irods/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/irods/connection.py b/irods/connection.py index 92255a0..c22ee23 100644 --- a/irods/connection.py +++ b/irods/connection.py @@ -50,7 +50,7 @@ def _connect(self): try: s.connect((self.account.host, self.account.port)) except socket.error: - raise Exception("Could not connect to specified host and port: %s:%s" % (self.account.host, self.account.post)) + raise Exception("Could not connect to specified host and port: %s:%s" % (self.account.host, self.account.port)) self.socket = s main_message = StartupPack( From 5a64b897abd30a75e3cd06a8aeb587359df1b1bb Mon Sep 17 00:00:00 2001 From: Chris La Rose Date: Tue, 14 Jan 2014 14:04:06 -0700 Subject: [PATCH 28/55] Added repr method for column --- irods/column.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/irods/column.py b/irods/column.py index f668be6..ad21a96 100644 --- a/irods/column.py +++ b/irods/column.py @@ -35,6 +35,14 @@ def __init__(self, type, icat_key, icat_id): self.icat_id = icat_id super(Column, self).__init__(type) + def __repr__(self): + return "<%s.%s %d %s>" % ( + self.__class__.__module__, + self.__class__.__name__, + self.icat_id, + self.icat_key + ) + class Keyword(QueryKey): def __init__(self, type, icat_key): self.icat_key = icat_key From 70188b945d3ad70a9fb74425417b7dde22b559be Mon Sep 17 00:00:00 2001 From: Chris La Rose Date: Tue, 14 Jan 2014 14:31:22 -0700 Subject: [PATCH 29/55] DataObjectManager.get now no longer fails when trying to get a data object that is replicated. DataObject now stores a list of four-tuples representing its replicas --- irods/data_object.py | 13 ++++++++++--- irods/resource_manager/data_object_manager.py | 7 +++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/irods/data_object.py b/irods/data_object.py index fdb86b1..a43a2bf 100644 --- a/irods/data_object.py +++ b/irods/data_object.py @@ -6,14 +6,21 @@ from irods.exception import CAT_NO_ACCESS_PERMISSION class iRODSDataObject(object): - def __init__(self, manager, parent=None, result=None): + def __init__(self, manager, parent=None, results=None): self.manager = manager - if parent and result: + if parent and results: self.collection = parent for attr in ['id', 'name', 'size', 'checksum', 'create_time', 'modify_time']: - setattr(self, attr, result[getattr(DataObject, attr)]) + setattr(self, attr, results[0][getattr(DataObject, attr)]) self.path = self.collection.path + '/' + self.name + replicas = sorted(results, key=lambda r: r[DataObject.replica_number]) + self.replicas = [( + r[DataObject.replica_status], + r[DataObject.resource_group_name], + r[DataObject.resource_name], + r[DataObject.path] + ) for r in replicas] self._meta = None def __repr__(self): diff --git a/irods/resource_manager/data_object_manager.py b/irods/resource_manager/data_object_manager.py index 9802974..437cd64 100644 --- a/irods/resource_manager/data_object_manager.py +++ b/irods/resource_manager/data_object_manager.py @@ -24,11 +24,10 @@ def get(self, path): query = self.sess.query(DataObject)\ .filter(DataObject.name == basename(path))\ .filter(DataObject.collection_id == parent.id) - try: - result = query.one() - except NoResultFound: + results = query.all() + if len(results) < 0: raise DataObjectDoesNotExist() - return iRODSDataObject(self, parent, result) + return iRODSDataObject(self, parent, results) def create(self, path): message_body = FileOpenRequest( From 7d6d3e4de7b8226b61150a545a2e83bb3fa58ff0 Mon Sep 17 00:00:00 2001 From: Chris La Rose Date: Tue, 14 Jan 2014 14:54:43 -0700 Subject: [PATCH 30/55] Getting data_objects of a collection where there exists replicas no longer returns duplicate data_objects --- irods/collection.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/irods/collection.py b/irods/collection.py index 350cdea..d72ae85 100644 --- a/irods/collection.py +++ b/irods/collection.py @@ -1,3 +1,5 @@ +import itertools +import operator from os.path import basename from irods.models import Collection, DataObject @@ -32,9 +34,10 @@ def data_objects(self): query = self.manager.sess.query(DataObject)\ .filter(DataObject.collection_id == self.id) results = query.all() + grouped = itertools.groupby(results, operator.itemgetter(DataObject.id)) return [ - iRODSDataObject(self.manager.sess.data_objects, self, row) - for row in results + iRODSDataObject(self.manager.sess.data_objects, self, list(replicas)) + for _, replicas in grouped ] def remove(self, recurse=True, force=False, additional_flags={}): From 81677edb00a9a4353a2b6627c0eb48d446a66d1c Mon Sep 17 00:00:00 2001 From: Chris La Rose Date: Tue, 14 Jan 2014 14:59:39 -0700 Subject: [PATCH 31/55] Added iRODSReplica class --- irods/data_object.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/irods/data_object.py b/irods/data_object.py index a43a2bf..64417c9 100644 --- a/irods/data_object.py +++ b/irods/data_object.py @@ -5,6 +5,21 @@ from irods.meta import iRODSMetaCollection from irods.exception import CAT_NO_ACCESS_PERMISSION +class iRODSReplica(object): + def __init__(self, status, resource_group_name, resource_name, path): + self.status = status + self.resource_group_name = resource_group_name + self.resource_name = resource_name + self.path = path + + def __repr__(self): + return "<%s.%s %s %s>" % ( + self.__class__.__module__, + self.__class__.__name__, + self.resource_group_name, + self.resource_name + ) + class iRODSDataObject(object): def __init__(self, manager, parent=None, results=None): self.manager = manager @@ -15,7 +30,7 @@ def __init__(self, manager, parent=None, results=None): setattr(self, attr, results[0][getattr(DataObject, attr)]) self.path = self.collection.path + '/' + self.name replicas = sorted(results, key=lambda r: r[DataObject.replica_number]) - self.replicas = [( + self.replicas = [iRODSReplica( r[DataObject.replica_status], r[DataObject.resource_group_name], r[DataObject.resource_name], From 4491b3b97b806e30ba23c10504a94497d5ecdf05 Mon Sep 17 00:00:00 2001 From: Chris La Rose Date: Tue, 14 Jan 2014 15:02:30 -0700 Subject: [PATCH 32/55] Upped version number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 23c9b60..3a83cfc 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from distutils.core import setup setup( name='python-irodsclient', - version='0.1', + version='0.2', author='Chris LaRose', author_email='cjlarose@iplantcollaborative.org', packages=['irods', 'irods.message', 'irods.resource_manager'], From 02dc22db21e220249b1865cb20916cc22e56b3d6 Mon Sep 17 00:00:00 2001 From: Chris LaRose Date: Tue, 8 Apr 2014 15:23:48 -0700 Subject: [PATCH 33/55] Destroying connections that encounter broken pipes. This makes connections more resiliant to connection resets by the iRODS host --- irods/connection.py | 13 +++++++++---- irods/exception.py | 3 +++ irods/pool.py | 8 +++++--- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/irods/connection.py b/irods/connection.py index c22ee23..4cc28d9 100644 --- a/irods/connection.py +++ b/irods/connection.py @@ -6,7 +6,7 @@ from irods.message import (iRODSMessage, StartupPack, AuthResponse, AuthChallenge, FileReadRequest, FileWriteRequest, FileSeekRequest, FileSeekResponse, FileCloseRequest) -from irods.exception import get_exception_by_code +from irods.exception import get_exception_by_code, NetworkException from irods import MAX_PASSWORD_LENGTH from irods.api_number import api_number @@ -27,7 +27,12 @@ def __del__(self): def send(self, message): str = message.pack() logger.debug(str) - return self.socket.sendall(str) + try: + self.socket.sendall(str) + except: + logger.error("Unable to send message. Connection to remote host may have closed. Releasing connection from pool.") + self.release(True) + raise NetworkException("Unable to send message") def recv(self): msg = iRODSMessage.recv(self.socket) @@ -41,8 +46,8 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): self.release() - def release(self): - self.pool.release_connection(self) + def release(self, destroy=False): + self.pool.release_connection(self, destroy) def _connect(self): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) diff --git a/irods/exception.py b/irods/exception.py index f7e88c1..ed424e1 100644 --- a/irods/exception.py +++ b/irods/exception.py @@ -4,6 +4,9 @@ class PycommandsException(Exception): pass +class NetworkException(PycommandsException): + pass + class DoesNotExist(PycommandsException): pass diff --git a/irods/pool.py b/irods/pool.py index d2bfcad..d6ebd93 100644 --- a/irods/pool.py +++ b/irods/pool.py @@ -22,8 +22,10 @@ def get_connection(self): logger.debug('num active: %d' % len(self.active)) return conn - def release_connection(self, conn): + def release_connection(self, conn, destroy=False): with self._lock: - self.active.remove(conn) - self.idle.add(conn) + if conn in self.active: + self.active.remove(conn) + if not destroy: + self.idle.add(conn) logger.debug('num idle: %d' % len(self.idle)) From fa87802a7c78be8faaff3b7faebc9d822d0b4f79 Mon Sep 17 00:00:00 2001 From: Chris LaRose Date: Tue, 8 Apr 2014 15:25:57 -0700 Subject: [PATCH 34/55] Upped to version 0.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3a83cfc..3c6fc99 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from distutils.core import setup setup( name='python-irodsclient', - version='0.2', + version='0.3', author='Chris LaRose', author_email='cjlarose@iplantcollaborative.org', packages=['irods', 'irods.message', 'irods.resource_manager'], From 48017aebec03b7bdfb4e6f3eef07531a0c4a3d90 Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Wed, 20 Aug 2014 12:17:43 -0500 Subject: [PATCH 35/55] Update to new API for collection creation --- irods/api_number.py | 2 +- irods/message/__init__.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/irods/api_number.py b/irods/api_number.py index ca8030b..da51dd9 100644 --- a/irods/api_number.py +++ b/irods/api_number.py @@ -150,7 +150,7 @@ "DATA_OBJ_WRITE201_AN": 604, "DATA_OBJ_CLOSE201_AN": 605, "DATA_OBJ_LSEEK201_AN": 612, - "COLL_CREATE201_AN": 616, + "COLL_CREATE201_AN": 681, "RM_COLL_OLD201_AN": 617, "REG_COLL201_AN": 618, "MOD_COLL201_AN": 646, diff --git a/irods/message/__init__.py b/irods/message/__init__.py index 9484f2d..142fd88 100644 --- a/irods/message/__init__.py +++ b/irods/message/__init__.py @@ -233,8 +233,10 @@ def __init__(self, *args): #define CollInp_PI "str collName[MAX_NAME_LEN]; struct KeyValPair_PI;" class CollectionRequest(Message): - _name = 'CollInp_PI' + _name = 'CollInpNew_PI' collName = StringProperty() + flags = IntegerProperty() + oprType = IntegerProperty() KeyValPair_PI = SubmessageProperty(StringStringMap) def empty_gen_query_out(cols): From 5d0ab6d77d385c42a90364bc1abce37191c0d385 Mon Sep 17 00:00:00 2001 From: Antoine de Torcy Date: Tue, 26 Aug 2014 18:52:39 -0400 Subject: [PATCH 36/55] [#3] WIP --- .gitignore | 5 +++++ irods/api_number.py | 2 +- irods/resource_manager/collection_manager.py | 4 ++-- irods/session.py | 11 +++++++++++ irods/test/collection_test.py | 4 ++-- setup.py | 4 ++-- 6 files changed, 23 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 247a900..e637beb 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,8 @@ nosetests.xml Session.vim .netrwhist *~ + +# virtualenv +testenv + + diff --git a/irods/api_number.py b/irods/api_number.py index da51dd9..01d3fed 100644 --- a/irods/api_number.py +++ b/irods/api_number.py @@ -150,7 +150,7 @@ "DATA_OBJ_WRITE201_AN": 604, "DATA_OBJ_CLOSE201_AN": 605, "DATA_OBJ_LSEEK201_AN": 612, - "COLL_CREATE201_AN": 681, + "COLL_CREATE_AN": 681, "RM_COLL_OLD201_AN": 617, "REG_COLL201_AN": 618, "MOD_COLL201_AN": 646, diff --git a/irods/resource_manager/collection_manager.py b/irods/resource_manager/collection_manager.py index 2f70f6b..fa44020 100644 --- a/irods/resource_manager/collection_manager.py +++ b/irods/resource_manager/collection_manager.py @@ -20,7 +20,7 @@ def create(self, path): KeyValPair_PI=StringStringMap() ) message = iRODSMessage('RODS_API_REQ', msg=message_body, - int_info=api_number['COLL_CREATE201_AN']) + int_info=api_number['COLL_CREATE_AN']) with self.sess.pool.get_connection() as conn: conn.send(message) response = conn.recv() @@ -38,7 +38,7 @@ def remove(self, path, recurse=True, force=False, additional_flags={}): KeyValPair_PI=StringStringMap(options) ) message = iRODSMessage('RODS_API_REQ', msg=message_body, - int_info=api_number['RM_COLL_OLD201_AN']) + int_info=api_number['RM_COLL_AN']) with self.sess.pool.get_connection() as conn: conn.send(message) response = conn.recv() diff --git a/irods/session.py b/irods/session.py index ebfd5d9..edb4a23 100644 --- a/irods/session.py +++ b/irods/session.py @@ -13,6 +13,17 @@ def __init__(self, *args, **kwargs): self.collections = CollectionManager(self) self.data_objects = DataObjectManager(self) self.metadata = MetadataManager(self) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.cleanup() + + def cleanup(self): + for conn in self.pool.active | self.pool.idle: + conn.disconnect() + conn.release(True) def configure(self, host=None, port=1247, user=None, zone=None, password=None, client_user=None, client_zone=None): diff --git a/irods/test/collection_test.py b/irods/test/collection_test.py index 7072b20..9b1f9e7 100644 --- a/irods/test/collection_test.py +++ b/irods/test/collection_test.py @@ -17,7 +17,7 @@ def setUp(self): password=config.IRODS_USER_PASSWORD, zone=config.IRODS_SERVER_ZONE) - self.coll = self.sess.create_collection(self.test_coll_path) + self.coll = self.sess.collections.create(self.test_coll_path) def tearDown(self): """ Delete the test collection after each test """ @@ -25,7 +25,7 @@ def tearDown(self): def test_get_collection(self): #path = "/tempZone/home/rods" - coll = self.sess.get_collection(self.test_coll_path) + coll = self.sess.collections.get(self.test_coll_path) self.assertEquals(self.test_coll_path, coll.path) #def test_new_collection(self): diff --git a/setup.py b/setup.py index 3c6fc99..62e27da 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,10 @@ -from distutils.core import setup +from setuptools import find_packages, setup setup( name='python-irodsclient', version='0.3', author='Chris LaRose', author_email='cjlarose@iplantcollaborative.org', - packages=['irods', 'irods.message', 'irods.resource_manager'], + packages=find_packages(), install_requires=[ 'PrettyTable>=0.7,<1.0' ] From 91d63181f9def671dd6dd4b59e68a5ff1e73e8e7 Mon Sep 17 00:00:00 2001 From: Antoine de Torcy Date: Fri, 29 Aug 2014 22:53:30 -0400 Subject: [PATCH 37/55] [#2] Updated API and packing instructions for FileCloseRequest() --- irods/connection.py | 12 +++++++++--- irods/message/__init__.py | 9 +++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/irods/connection.py b/irods/connection.py index 4cc28d9..6469142 100644 --- a/irods/connection.py +++ b/irods/connection.py @@ -5,7 +5,7 @@ from irods.message import (iRODSMessage, StartupPack, AuthResponse, AuthChallenge, FileReadRequest, FileWriteRequest, FileSeekRequest, FileSeekResponse, - FileCloseRequest) + FileCloseRequest, StringStringMap) from irods.exception import get_exception_by_code, NetworkException from irods import MAX_PASSWORD_LENGTH from irods.api_number import api_number @@ -135,10 +135,16 @@ def seek_file(self, desc, offset, whence): def close_file(self, desc): message_body = FileCloseRequest( - l1descInx=desc + l1descInx=desc, + len=0, + whence=0, + oprType=0, + offset=0, + bytesWritten=0, + KeyValPair_PI=StringStringMap() ) message = iRODSMessage('RODS_API_REQ', msg=message_body, - int_info=api_number['DATA_OBJ_CLOSE201_AN']) + int_info=api_number['DATA_OBJ_CLOSE_AN']) self.send(message) response = self.recv() diff --git a/irods/message/__init__.py b/irods/message/__init__.py index 142fd88..8e35db1 100644 --- a/irods/message/__init__.py +++ b/irods/message/__init__.py @@ -205,11 +205,16 @@ class FileSeekResponse(Message): _name = 'fileLseekOut_PI' offset = LongProperty() -#define dataObjCloseInp_PI "int l1descInx; double bytesWritten;" +#define OpenedDataObjInp_PI "int l1descInx; int len; int whence; int oprType; double offset; double bytesWritten; struct KeyValPair_PI;" class FileCloseRequest(Message): - _name = 'dataObjCloseInp_PI' + _name = 'OpenedDataObjInp_PI' l1descInx = IntegerProperty() + len = IntegerProperty() + whence = IntegerProperty() + oprType = IntegerProperty() + offset = LongProperty() bytesWritten = LongProperty() + KeyValPair_PI = SubmessageProperty(StringStringMap) #define ModAVUMetadataInp_PI "str *arg0; str *arg1; str *arg2; str *arg3; str *arg4; str *arg5; str *arg6; str *arg7; str *arg8; str *arg9;" class MetadataRequest(Message): From 192bc50c4b69e18b05e824fe038d8978b4e00933 Mon Sep 17 00:00:00 2001 From: Antoine de Torcy Date: Sun, 31 Aug 2014 03:44:07 -0400 Subject: [PATCH 38/55] [#3] Fixes for connection_test.py, collection_test.py and file_test.py --- irods/collection.py | 3 ++ irods/connection.py | 34 ++++++++++++++------ irods/message/__init__.py | 32 ++++++++++++------ irods/resource_manager/collection_manager.py | 3 +- irods/test/collection_test.py | 4 ++- irods/test/connection_test.py | 7 +++- irods/test/file_test.py | 26 +++++++++++++-- 7 files changed, 83 insertions(+), 26 deletions(-) diff --git a/irods/collection.py b/irods/collection.py index d72ae85..a2bb5bb 100644 --- a/irods/collection.py +++ b/irods/collection.py @@ -42,6 +42,9 @@ def data_objects(self): def remove(self, recurse=True, force=False, additional_flags={}): self.manager.remove(self.path, recurse, force, additional_flags) + + def move(self, path): + pass def walk(self, topdown=True): """ diff --git a/irods/connection.py b/irods/connection.py index 6469142..0e19850 100644 --- a/irods/connection.py +++ b/irods/connection.py @@ -96,11 +96,16 @@ def _login(self): def read_file(self, desc, size): message_body = FileReadRequest( - l1descInx=desc, - len=size + l1descInx=desc, + len=size, + whence=0, + oprType=0, + offset=0, + bytesWritten=0, + KeyValPair_PI=StringStringMap() ) message = iRODSMessage('RODS_API_REQ', msg=message_body, - int_info=api_number['DATA_OBJ_READ201_AN']) + int_info=api_number['DATA_OBJ_READ_AN']) logger.debug(desc) self.send(message) @@ -109,24 +114,33 @@ def read_file(self, desc, size): def write_file(self, desc, string): message_body = FileWriteRequest( - dataObjInx=desc, - len=len(string) + l1descInx=desc, + len=len(string), + whence=0, + oprType=0, + offset=0, + bytesWritten=0, + KeyValPair_PI=StringStringMap() ) message = iRODSMessage('RODS_API_REQ', msg=message_body, bs=string, - int_info=api_number['DATA_OBJ_WRITE201_AN']) + int_info=api_number['DATA_OBJ_WRITE_AN']) self.send(message) response = self.recv() return response.int_info def seek_file(self, desc, offset, whence): message_body = FileSeekRequest( - fileInx=desc, - offset=offset, - whence=whence + l1descInx=desc, + len=0, + whence=whence, + oprType=0, + offset=offset, + bytesWritten=0, + KeyValPair_PI=StringStringMap() ) message = iRODSMessage('RODS_API_REQ', msg=message_body, - int_info=api_number['DATA_OBJ_LSEEK201_AN']) + int_info=api_number['DATA_OBJ_LSEEK_AN']) self.send(message) response = self.recv() diff --git a/irods/message/__init__.py b/irods/message/__init__.py index 8e35db1..9cc45a2 100644 --- a/irods/message/__init__.py +++ b/irods/message/__init__.py @@ -181,24 +181,38 @@ class FileOpenRequest(Message): oprType = IntegerProperty() KeyValPair_PI = SubmessageProperty(StringStringMap) -#define dataObjReadInp_PI "int l1descInx; int len;" +#define OpenedDataObjInp_PI "int l1descInx; int len; int whence; int oprType; double offset; double bytesWritten; struct KeyValPair_PI;" class FileReadRequest(Message): - _name = 'dataObjReadInp_PI' + _name = 'OpenedDataObjInp_PI' l1descInx = IntegerProperty() len = IntegerProperty() + whence = IntegerProperty() + oprType = IntegerProperty() + offset = LongProperty() + bytesWritten = LongProperty() + KeyValPair_PI = SubmessageProperty(StringStringMap) -#define dataObjWriteInp_PI "int dataObjInx; int len;" +#define OpenedDataObjInp_PI "int l1descInx; int len; int whence; int oprType; double offset; double bytesWritten; struct KeyValPair_PI;" class FileWriteRequest(Message): - _name = 'dataObjWriteInp_PI' - dataObjInx = IntegerProperty() + _name = 'OpenedDataObjInp_PI' + l1descInx = IntegerProperty() len = IntegerProperty() + whence = IntegerProperty() + oprType = IntegerProperty() + offset = LongProperty() + bytesWritten = LongProperty() + KeyValPair_PI = SubmessageProperty(StringStringMap) -#define fileLseekInp_PI "int fileInx; double offset; int whence" +#define OpenedDataObjInp_PI "int l1descInx; int len; int whence; int oprType; double offset; double bytesWritten; struct KeyValPair_PI;" class FileSeekRequest(Message): - _name = 'fileLseekInp_PI' - fileInx = IntegerProperty() - offset = LongProperty() + _name = 'OpenedDataObjInp_PI' + l1descInx = IntegerProperty() + len = IntegerProperty() whence = IntegerProperty() + oprType = IntegerProperty() + offset = LongProperty() + bytesWritten = LongProperty() + KeyValPair_PI = SubmessageProperty(StringStringMap) #define fileLseekOut_PI "double offset;" class FileSeekResponse(Message): diff --git a/irods/resource_manager/collection_manager.py b/irods/resource_manager/collection_manager.py index fa44020..481876d 100644 --- a/irods/resource_manager/collection_manager.py +++ b/irods/resource_manager/collection_manager.py @@ -43,5 +43,4 @@ def remove(self, path, recurse=True, force=False, additional_flags={}): conn.send(message) response = conn.recv() - def move(self, path): - pass + diff --git a/irods/test/collection_test.py b/irods/test/collection_test.py index 9b1f9e7..0ee2e56 100644 --- a/irods/test/collection_test.py +++ b/irods/test/collection_test.py @@ -21,7 +21,8 @@ def setUp(self): def tearDown(self): """ Delete the test collection after each test """ - self.coll.remove() + self.coll.remove(recurse=True, force=True) + self.sess.cleanup() def test_get_collection(self): #path = "/tempZone/home/rods" @@ -43,6 +44,7 @@ def test_update_in_collection(self): """ Modify a file in a collection """ pass + @unittest.skip('Renaming collections is not yet implemented') def test_move_collection(self): new_path = "/tempZone/home/rods/test_dir_moved" self.coll.move(new_path) diff --git a/irods/test/connection_test.py b/irods/test/connection_test.py index 952cd81..710623a 100644 --- a/irods/test/connection_test.py +++ b/irods/test/connection_test.py @@ -17,6 +17,11 @@ def setUp(self): user=config.IRODS_USER_USERNAME, password=config.IRODS_USER_PASSWORD, zone=config.IRODS_SERVER_ZONE) + + def tearDown(self): + '''Close connections + ''' + self.sess.cleanup() def test_connection(self): """ @@ -24,7 +29,7 @@ def test_connection(self): There should be a better way to test this... Wouldn't the iRODSSession init establish the connection? """ - coll = self.sess.get_collection('/tempZone/home/rods') + coll = self.sess.collections.get('/tempZone/home/rods') self.assertTrue(coll, "Connection failed.") @unittest.skip("unimplemented") diff --git a/irods/test/file_test.py b/irods/test/file_test.py index 23a428c..3927461 100644 --- a/irods/test/file_test.py +++ b/irods/test/file_test.py @@ -7,6 +7,11 @@ class TestFiles(unittest.TestCase): """ """ + test_coll_path = '/tempZone/home/rods/test_dir' + test_obj_name = 'test1' + test_content_str = 'blah' + + test_obj_path = test_coll_path + '/' + test_obj_name def setUp(self): from irods.session import iRODSSession @@ -17,11 +22,26 @@ def setUp(self): user=config.IRODS_USER_USERNAME, password=config.IRODS_USER_PASSWORD, zone=config.IRODS_SERVER_ZONE) + + # Create test collection + self.test_coll = self.sess.collections.create(self.test_coll_path) + + # Create test object and write to it + self.test_obj = self.sess.data_objects.create(self.test_obj_path) + with self.test_obj.open('r+') as f: + f.write(self.test_content_str) + + + def tearDown(self): + '''Remove test data and close connections + ''' + self.test_coll.remove(recurse=True, force=True) + self.sess.cleanup() def test_file_open(self): #from irods.models import Collection, User, DataObject - obj = self.sess.get_data_object("/tempZone/home/rods/test1") + obj = self.sess.data_objects.get(self.test_obj_path) f = obj.open('r+') f.seek(0, 0) # what does this return? str1 = f.read() @@ -31,7 +51,7 @@ def test_file_open(self): def test_file_read(self): #from irods.models import Collection, User, DataObject - obj = self.sess.get_data_object("/tempZone/home/rods/test1") + obj = self.sess.data_objects.get(self.test_obj_path) f = obj.open('r+') str1 = f.read(1024) #self.assertTrue(expr, msg) @@ -40,7 +60,7 @@ def test_file_read(self): def test_file_write(self): #from irods.models import Collection, User, DataObject - obj = self.sess.get_data_object("/tempZone/home/rods/test1") + obj = self.sess.data_objects.get(self.test_obj_path) f = obj.open('w+') f.write("NEW STRING.py") f.seek(-6, 2) From 9d530509eea42746e374ced700cd52ab432c8476 Mon Sep 17 00:00:00 2001 From: Antoine de Torcy Date: Sun, 31 Aug 2014 03:56:30 -0400 Subject: [PATCH 39/55] [#3] Removed redundant code --- irods/connection.py | 11 +++++------ irods/message/__init__.py | 2 +- irods/resource_manager/data_object_manager.py | 7 ++----- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/irods/connection.py b/irods/connection.py index 0e19850..2b73d25 100644 --- a/irods/connection.py +++ b/irods/connection.py @@ -4,8 +4,7 @@ import hashlib from irods.message import (iRODSMessage, StartupPack, AuthResponse, AuthChallenge, - FileReadRequest, FileWriteRequest, FileSeekRequest, FileSeekResponse, - FileCloseRequest, StringStringMap) + OpenedDataObjRequest, FileSeekResponse, StringStringMap) from irods.exception import get_exception_by_code, NetworkException from irods import MAX_PASSWORD_LENGTH from irods.api_number import api_number @@ -95,7 +94,7 @@ def _login(self): auth_response = self.recv() def read_file(self, desc, size): - message_body = FileReadRequest( + message_body = OpenedDataObjRequest( l1descInx=desc, len=size, whence=0, @@ -113,7 +112,7 @@ def read_file(self, desc, size): return response.bs def write_file(self, desc, string): - message_body = FileWriteRequest( + message_body = OpenedDataObjRequest( l1descInx=desc, len=len(string), whence=0, @@ -130,7 +129,7 @@ def write_file(self, desc, string): return response.int_info def seek_file(self, desc, offset, whence): - message_body = FileSeekRequest( + message_body = OpenedDataObjRequest( l1descInx=desc, len=0, whence=whence, @@ -148,7 +147,7 @@ def seek_file(self, desc, offset, whence): return offset def close_file(self, desc): - message_body = FileCloseRequest( + message_body = OpenedDataObjRequest( l1descInx=desc, len=0, whence=0, diff --git a/irods/message/__init__.py b/irods/message/__init__.py index 9cc45a2..77f25d1 100644 --- a/irods/message/__init__.py +++ b/irods/message/__init__.py @@ -182,7 +182,7 @@ class FileOpenRequest(Message): KeyValPair_PI = SubmessageProperty(StringStringMap) #define OpenedDataObjInp_PI "int l1descInx; int len; int whence; int oprType; double offset; double bytesWritten; struct KeyValPair_PI;" -class FileReadRequest(Message): +class OpenedDataObjRequest(Message): _name = 'OpenedDataObjInp_PI' l1descInx = IntegerProperty() len = IntegerProperty() diff --git a/irods/resource_manager/data_object_manager.py b/irods/resource_manager/data_object_manager.py index 437cd64..b5c5231 100644 --- a/irods/resource_manager/data_object_manager.py +++ b/irods/resource_manager/data_object_manager.py @@ -2,11 +2,8 @@ from irods.models import DataObject from irods.resource_manager import ResourceManager -from irods.message import (iRODSMessage, FileReadRequest, FileWriteRequest, - FileSeekRequest, FileSeekResponse, FileOpenRequest, FileCloseRequest, - StringStringMap) -from irods.exception import (DataObjectDoesNotExist, CollectionDoesNotExist, - NoResultFound) +from irods.message import (iRODSMessage, FileOpenRequest, StringStringMap) +from irods.exception import (DataObjectDoesNotExist, CollectionDoesNotExist) from irods.api_number import api_number from irods.data_object import iRODSDataObject From bab463e09f01954de686f6af55eee80cc50f6dc4 Mon Sep 17 00:00:00 2001 From: Antoine de Torcy Date: Sun, 31 Aug 2014 03:59:04 -0400 Subject: [PATCH 40/55] [#3] Removed redundant code (continued) --- irods/message/__init__.py | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/irods/message/__init__.py b/irods/message/__init__.py index 77f25d1..a3a9dd5 100644 --- a/irods/message/__init__.py +++ b/irods/message/__init__.py @@ -192,44 +192,11 @@ class OpenedDataObjRequest(Message): bytesWritten = LongProperty() KeyValPair_PI = SubmessageProperty(StringStringMap) -#define OpenedDataObjInp_PI "int l1descInx; int len; int whence; int oprType; double offset; double bytesWritten; struct KeyValPair_PI;" -class FileWriteRequest(Message): - _name = 'OpenedDataObjInp_PI' - l1descInx = IntegerProperty() - len = IntegerProperty() - whence = IntegerProperty() - oprType = IntegerProperty() - offset = LongProperty() - bytesWritten = LongProperty() - KeyValPair_PI = SubmessageProperty(StringStringMap) - -#define OpenedDataObjInp_PI "int l1descInx; int len; int whence; int oprType; double offset; double bytesWritten; struct KeyValPair_PI;" -class FileSeekRequest(Message): - _name = 'OpenedDataObjInp_PI' - l1descInx = IntegerProperty() - len = IntegerProperty() - whence = IntegerProperty() - oprType = IntegerProperty() - offset = LongProperty() - bytesWritten = LongProperty() - KeyValPair_PI = SubmessageProperty(StringStringMap) - #define fileLseekOut_PI "double offset;" class FileSeekResponse(Message): _name = 'fileLseekOut_PI' offset = LongProperty() -#define OpenedDataObjInp_PI "int l1descInx; int len; int whence; int oprType; double offset; double bytesWritten; struct KeyValPair_PI;" -class FileCloseRequest(Message): - _name = 'OpenedDataObjInp_PI' - l1descInx = IntegerProperty() - len = IntegerProperty() - whence = IntegerProperty() - oprType = IntegerProperty() - offset = LongProperty() - bytesWritten = LongProperty() - KeyValPair_PI = SubmessageProperty(StringStringMap) - #define ModAVUMetadataInp_PI "str *arg0; str *arg1; str *arg2; str *arg3; str *arg4; str *arg5; str *arg6; str *arg7; str *arg8; str *arg9;" class MetadataRequest(Message): _name = 'ModAVUMetadataInp_PI' From 988238b54780b4c2bffe03c4772d0396fab5af48 Mon Sep 17 00:00:00 2001 From: Antoine de Torcy Date: Sun, 31 Aug 2014 05:10:07 -0400 Subject: [PATCH 41/55] [#3] Refined file IO tests a bit --- irods/test/file_test.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/irods/test/file_test.py b/irods/test/file_test.py index 3927461..e4e3f79 100644 --- a/irods/test/file_test.py +++ b/irods/test/file_test.py @@ -9,7 +9,9 @@ class TestFiles(unittest.TestCase): """ test_coll_path = '/tempZone/home/rods/test_dir' test_obj_name = 'test1' - test_content_str = 'blah' + content_str = 'blah' + write_str = '0123456789' + write_str1 = 'INTERRUPT' test_obj_path = test_coll_path + '/' + test_obj_name @@ -29,7 +31,7 @@ def setUp(self): # Create test object and write to it self.test_obj = self.sess.data_objects.create(self.test_obj_path) with self.test_obj.open('r+') as f: - f.write(self.test_content_str) + f.write(self.content_str) def tearDown(self): @@ -44,7 +46,11 @@ def test_file_open(self): obj = self.sess.data_objects.get(self.test_obj_path) f = obj.open('r+') f.seek(0, 0) # what does this return? - str1 = f.read() + + # for lack of anything better... + assert (f.tell() == 0) + + #str1 = f.read() #self.assertTrue(expr, msg) f.close() @@ -55,6 +61,10 @@ def test_file_read(self): f = obj.open('r+') str1 = f.read(1024) #self.assertTrue(expr, msg) + + # check content of test file + assert(str1 == self.content_str) + f.close() def test_file_write(self): @@ -62,9 +72,17 @@ def test_file_write(self): obj = self.sess.data_objects.get(self.test_obj_path) f = obj.open('w+') - f.write("NEW STRING.py") + f.write(self.write_str) f.seek(-6, 2) - f.write("INTERRUPT") + f.write(self.write_str1) + + # reset stream position for reading + f.seek(0, 0) + + # check new content of file after our write + str1 = f.read(1024) + assert(str1 == (self.write_str[:-6] + self.write_str1)) + #self.assertTrue(expr, msg) f.close() From 4ed9f96b569401a6515b8095f85cdd6a49a042cd Mon Sep 17 00:00:00 2001 From: Antoine de Torcy Date: Mon, 1 Sep 2014 17:40:02 -0400 Subject: [PATCH 42/55] [#3] Fix for message_test.py --- irods/test/message_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/irods/test/message_test.py b/irods/test/message_test.py index 020ebfb..dfa10f5 100644 --- a/irods/test/message_test.py +++ b/irods/test/message_test.py @@ -16,7 +16,7 @@ class TestMessages(unittest.TestCase): def test_startup_pack(self): - sup = StartupPack() + sup = StartupPack(('rods', 'tempZone'), ('rods', 'tempZone')) sup.irodsProt = 2 sup.reconnFlag = 3 sup.proxyUser = "rods" @@ -30,6 +30,7 @@ def test_startup_pack(self): expected = "\ 2\ 3\ +0\ rods\ tempZone\ rods\ @@ -40,7 +41,7 @@ def test_startup_pack(self): " self.assertEqual(xml_str, expected) - sup2 = StartupPack() + sup2 = StartupPack(('rods', 'tempZone'), ('rods', 'tempZone')) sup2.unpack(ET.fromstring(expected)) self.assertEqual(sup2.irodsProt, 2) self.assertEqual(sup2.reconnFlag, 3) From 9d4261a0a56b3482042a48adb7b06b608bcc86d6 Mon Sep 17 00:00:00 2001 From: Antoine de Torcy Date: Mon, 1 Sep 2014 18:44:42 -0400 Subject: [PATCH 43/55] [#3] Fix for meta_test.py (WIP) --- irods/resource_manager/metadata_manager.py | 2 +- irods/test/meta_test.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/irods/resource_manager/metadata_manager.py b/irods/resource_manager/metadata_manager.py index 18938cd..613a5c1 100644 --- a/irods/resource_manager/metadata_manager.py +++ b/irods/resource_manager/metadata_manager.py @@ -17,7 +17,7 @@ def _model_class_to_resource_type(model_cls): DataObject: 'd', Collection: 'c', Resource: 'r', - User: 'r', + User: 'u', }[model_cls] def get(self, model_cls, path): diff --git a/irods/test/meta_test.py b/irods/test/meta_test.py index a96f034..cf6f9e6 100644 --- a/irods/test/meta_test.py +++ b/irods/test/meta_test.py @@ -2,6 +2,8 @@ import unittest import os import sys +from irods.models import (DataObject, Collection, Resource, User, DataObjectMeta, + CollectionMeta, ResourceMeta, UserMeta) class TestMeta(unittest.TestCase): @@ -17,6 +19,12 @@ def setUp(self): user=config.IRODS_USER_USERNAME, password=config.IRODS_USER_PASSWORD, zone=config.IRODS_SERVER_ZONE) + + def tearDown(self): + '''Remove test data and close connections + ''' + #self.test_coll.remove(recurse=True, force=True) + self.sess.cleanup() def test_get_meta(self): """ @@ -24,10 +32,11 @@ def test_get_meta(self): #from irods.meta import iRODSMeta #obj = self.sess.get_data_object("/tempZone/home/rods/test1") - meta = self.sess.get_meta('d', "/tempZone/home/rods/test1") + meta = self.sess.metadata.get(DataObject, '/tempZone/home/rods/test1') print meta #self.assertEqual(first, second, msg) + @unittest.skip('') def test_add_meta(self): """ """ @@ -36,6 +45,7 @@ def test_add_meta(self): self.sess.add_meta('d', '/tempZone/home/rods/test1', iRODSMeta('key8', 'value5')) + @unittest.skip('') def test_copy_meta(self): """ """ @@ -43,6 +53,7 @@ def test_copy_meta(self): self.sess.copy_meta('d', 'd', '/tempZone/home/rods/test1', '/tempZone/home/rods/test2') + @unittest.skip('') def test_remove_meta(self): """ """ From ce63e9668e291f9a68bbadbca80b6977a9d5761e Mon Sep 17 00:00:00 2001 From: Antoine de Torcy Date: Thu, 4 Sep 2014 11:40:30 -0400 Subject: [PATCH 44/55] [#5] comments --- irods/test/file_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/irods/test/file_test.py b/irods/test/file_test.py index e4e3f79..7ef7f8c 100644 --- a/irods/test/file_test.py +++ b/irods/test/file_test.py @@ -5,8 +5,8 @@ class TestFiles(unittest.TestCase): - """ - """ + '''Suite of data object I/O unit tests + ''' test_coll_path = '/tempZone/home/rods/test_dir' test_obj_name = 'test1' content_str = 'blah' From d70b442ecb91ec866d650dd7e2d7d2fdbe6993a7 Mon Sep 17 00:00:00 2001 From: Antoine de Torcy Date: Thu, 4 Sep 2014 11:41:43 -0400 Subject: [PATCH 45/55] [#3] Fix for meta_test.py --- irods/test/meta_test.py | 106 +++++++++++++++++++++++++++++++--------- 1 file changed, 82 insertions(+), 24 deletions(-) diff --git a/irods/test/meta_test.py b/irods/test/meta_test.py index cf6f9e6..dfe7ac3 100644 --- a/irods/test/meta_test.py +++ b/irods/test/meta_test.py @@ -2,13 +2,26 @@ import unittest import os import sys +from irods.meta import iRODSMeta from irods.models import (DataObject, Collection, Resource, User, DataObjectMeta, CollectionMeta, ResourceMeta, UserMeta) + + class TestMeta(unittest.TestCase): - """ - """ + '''Suite of tests on metadata operations + ''' + + # test data + coll_path = '/tempZone/home/rods/test_dir' + obj_name = 'test1' + obj_path = coll_path + '/' + obj_name + + # test metadata + (attr0, value0, unit0) = ('attr0', 'value0', 'unit0') + (attr1, value1, unit1) = ('attr1', 'value1', 'unit1') + def setUp(self): from irods.session import iRODSSession @@ -20,47 +33,92 @@ def setUp(self): password=config.IRODS_USER_PASSWORD, zone=config.IRODS_SERVER_ZONE) + # Create test collection and (empty) test object + self.coll = self.sess.collections.create(self.coll_path) + self.obj = self.sess.data_objects.create(self.obj_path) + + def tearDown(self): '''Remove test data and close connections ''' - #self.test_coll.remove(recurse=True, force=True) + self.coll.remove(recurse=True, force=True) self.sess.cleanup() - def test_get_meta(self): + def test_get_obj_meta(self): """ """ - #from irods.meta import iRODSMeta - - #obj = self.sess.get_data_object("/tempZone/home/rods/test1") - meta = self.sess.metadata.get(DataObject, '/tempZone/home/rods/test1') - print meta + # get object metadata + meta = self.sess.metadata.get(DataObject, self.obj_path) + + # there should be no metadata at this point + assert (len(meta) == 0) + #self.assertEqual(first, second, msg) - @unittest.skip('') - def test_add_meta(self): + + def test_add_obj_meta(self): """ """ - from irods.meta import iRODSMeta - self.sess.add_meta('d', '/tempZone/home/rods/test1', - iRODSMeta('key8', 'value5')) + # add metadata to test object + self.sess.metadata.add(DataObject, self.obj_path, + iRODSMeta(self.attr0, self.value0)) + self.sess.metadata.add(DataObject, self.obj_path, + iRODSMeta(self.attr1, self.value1, self.unit1)) + + # get object metadata + meta = self.sess.metadata.get(DataObject, self.obj_path) + + # assertions + assert(meta[0].name == self.attr0) + assert(meta[0].value == self.value0) + + assert(meta[1].name == self.attr1) + assert(meta[1].value == self.value1) + assert(meta[1].units == self.unit1) - @unittest.skip('') - def test_copy_meta(self): + + def test_copy_obj_meta(self): """ """ - #from irods.meta import iRODSMeta - self.sess.copy_meta('d', 'd', '/tempZone/home/rods/test1', - '/tempZone/home/rods/test2') + + # test destination object for copy + dest_obj_path = self.coll_path + '/test2' + self.sess.data_objects.create(dest_obj_path) + + # add metadata to test object + self.sess.metadata.add(DataObject, self.obj_path, + iRODSMeta(self.attr0, self.value0)) + + # copy metadata + self.sess.metadata.copy(DataObject, DataObject, self.obj_path, dest_obj_path) + + # get destination object metadata + dest_meta = self.sess.metadata.get(DataObject, dest_obj_path) + + # check metadata + assert(dest_meta[0].name == self.attr0) + - @unittest.skip('') - def test_remove_meta(self): + def test_remove_obj_meta(self): """ """ - from irods.meta import iRODSMeta + + # add metadata to test object + self.sess.metadata.add(DataObject, self.obj_path, + iRODSMeta(self.attr0, self.value0)) + + # check that metadata is there + meta = self.sess.metadata.get(DataObject, self.obj_path) + assert(meta[0].name == self.attr0) - self.sess.remove_meta('d', '/tempZone/home/rods/test1', - iRODSMeta('key8', 'value5')) + # remove metadata from object + self.sess.metadata.remove(DataObject, self.obj_path, + iRODSMeta(self.attr0, self.value0)) + + # check that metadata is gone + meta = self.sess.metadata.get(DataObject, self.obj_path) + assert (len(meta) == 0) if __name__ == '__main__': From 1bb53e178e9e9a27551b0071922f6cae801df194 Mon Sep 17 00:00:00 2001 From: Antoine de Torcy Date: Thu, 4 Sep 2014 18:00:25 -0400 Subject: [PATCH 46/55] [#3] Fix for query_test.py --- irods/results.py | 11 ++++++ irods/test/meta_test.py | 2 +- irods/test/query_test.py | 73 ++++++++++++++++++++++++++-------------- 3 files changed, 60 insertions(+), 26 deletions(-) diff --git a/irods/results.py b/irods/results.py index 7b01b72..60422b2 100644 --- a/irods/results.py +++ b/irods/results.py @@ -34,3 +34,14 @@ def __iter__(self): def __len__(self): return self.length + + # For testing. Might go somewhere else... + def has_value(self, value): + found = False + + for row in self.rows: + if value in row.values(): + found = True + + return found + diff --git a/irods/test/meta_test.py b/irods/test/meta_test.py index dfe7ac3..7023908 100644 --- a/irods/test/meta_test.py +++ b/irods/test/meta_test.py @@ -16,7 +16,7 @@ class TestMeta(unittest.TestCase): # test data coll_path = '/tempZone/home/rods/test_dir' obj_name = 'test1' - obj_path = coll_path + '/' + obj_name + obj_path = '{0}/{1}'.format(coll_path, obj_name) # test metadata (attr0, value0, unit0) = ('attr0', 'value0', 'unit0') diff --git a/irods/test/query_test.py b/irods/test/query_test.py index 559b0e8..a9f374c 100644 --- a/irods/test/query_test.py +++ b/irods/test/query_test.py @@ -2,11 +2,17 @@ import unittest import os import sys +from irods.models import User, Collection, Keywords, DataObject class TestQuery(unittest.TestCase): """ """ + # test data + coll_path = '/tempZone/home/rods/test_dir' + obj_name = 'test1' + obj_path = '{0}/{1}'.format(coll_path, obj_name) + def setUp(self): from irods.session import iRODSSession @@ -17,33 +23,44 @@ def setUp(self): user=config.IRODS_USER_USERNAME, password=config.IRODS_USER_PASSWORD, zone=config.IRODS_SERVER_ZONE) - - def test_query(self): - #from irods.query import Query - from irods.models import User, Collection, Keywords - - q1 = self.sess.query(User, Collection.name) - q2 = q1.filter(User.name == 'cjlarose') - q3 = q2.filter(Keywords.chksum == '12345') - - f = open('select', 'w') - f.write(q3._select_message().pack()) - - f = open('conds', 'w') - f.write(q3._conds_message().pack()) - - f = open('condskw', 'w') - f.write(q3._kw_message().pack()) - - f = open('genq', 'w') - f.write(q3._message().pack()) - - self.sess.query(Collection.id, Collection.name).all() + + # Create test collection and (empty) test object + self.coll = self.sess.collections.create(self.coll_path) + self.obj = self.sess.data_objects.create(self.obj_path) + + def tearDown(self): + '''Remove test data and close connections + ''' + self.coll.remove(recurse=True, force=True) + self.sess.cleanup() + + + def test_collections_query(self): + # collection query test + result = self.sess.query(Collection.id, Collection.name).all() + assert ( result.has_value(self.coll_path) ) + +# q1 = self.sess.query(User, Collection.name) +# q2 = q1.filter(User.name == 'cjlarose') +# q3 = q2.filter(Keywords.chksum == '12345') +# +# f = open('select', 'w') +# f.write(q1._select_message().pack()) +# +# f = open('conds', 'w') +# f.write(q1._conds_message().pack()) +# +# f = open('condskw', 'w') +# f.write(q1._kw_message().pack()) +# +# f = open('genq', 'w') +# f.write(q1._message().pack()) + + #print result """ cut-n-pasted from collection_test... """ - from irods.models import Collection, User, DataObject #q1 = sess.query(Collection.id).filter(Collection.name == "'/tempZone/home/rods'") #q1.all() @@ -54,10 +71,16 @@ def test_query(self): #result = sess.query(Collection.id, Collection.owner_name, User.id, User.name)\ # .filter(Collection.owner_name == "'rods'")\ # .all() - + + #print result + + + + def test_files_query(self): + # file query test result = self.sess.query(DataObject.id, DataObject.collection_id, DataObject.name, User.name, Collection.name).all() + assert ( result.has_value(self.obj_name) ) - print str(result) if __name__ == '__main__': From bfbf52b49edb730b6a517825c7dda585e15926a4 Mon Sep 17 00:00:00 2001 From: Antoine de Torcy Date: Fri, 5 Sep 2014 17:54:12 -0400 Subject: [PATCH 47/55] [#5] Use test credentials from config module --- irods/test/collection_test.py | 7 +++---- irods/test/connection_test.py | 7 +++---- irods/test/file_test.py | 7 +++---- irods/test/meta_test.py | 7 +++---- irods/test/query_test.py | 7 +++---- 5 files changed, 15 insertions(+), 20 deletions(-) diff --git a/irods/test/collection_test.py b/irods/test/collection_test.py index 0ee2e56..b652760 100644 --- a/irods/test/collection_test.py +++ b/irods/test/collection_test.py @@ -2,15 +2,14 @@ import os import sys import unittest +from irods.session import iRODSSession +import config class TestCollection(unittest.TestCase): - test_coll_path = "/tempZone/home/rods/test_dir" + test_coll_path = '/{0}/home/{1}/test_dir'.format(config.IRODS_SERVER_ZONE, config.IRODS_USER_USERNAME) def setUp(self): - from irods.session import iRODSSession - import config - self.sess = iRODSSession(host=config.IRODS_SERVER_HOST, port=config.IRODS_SERVER_PORT, user=config.IRODS_USER_USERNAME, diff --git a/irods/test/connection_test.py b/irods/test/connection_test.py index 710623a..ed7f217 100644 --- a/irods/test/connection_test.py +++ b/irods/test/connection_test.py @@ -2,6 +2,8 @@ import unittest import os import sys +from irods.session import iRODSSession +import config class TestConnections(unittest.TestCase): @@ -9,9 +11,6 @@ class TestConnections(unittest.TestCase): """ def setUp(self): - from irods.session import iRODSSession - import config - self.sess = iRODSSession(host=config.IRODS_SERVER_HOST, port=config.IRODS_SERVER_PORT, # 4444: why? user=config.IRODS_USER_USERNAME, @@ -29,7 +28,7 @@ def test_connection(self): There should be a better way to test this... Wouldn't the iRODSSession init establish the connection? """ - coll = self.sess.collections.get('/tempZone/home/rods') + coll = self.sess.collections.get('/{0}/home/{1}'.format(config.IRODS_SERVER_ZONE, config.IRODS_USER_USERNAME)) self.assertTrue(coll, "Connection failed.") @unittest.skip("unimplemented") diff --git a/irods/test/file_test.py b/irods/test/file_test.py index 7ef7f8c..22bf052 100644 --- a/irods/test/file_test.py +++ b/irods/test/file_test.py @@ -2,12 +2,14 @@ import unittest import os import sys +from irods.session import iRODSSession +import config class TestFiles(unittest.TestCase): '''Suite of data object I/O unit tests ''' - test_coll_path = '/tempZone/home/rods/test_dir' + test_coll_path = '/{0}/home/{1}/test_dir'.format(config.IRODS_SERVER_ZONE, config.IRODS_USER_USERNAME) test_obj_name = 'test1' content_str = 'blah' write_str = '0123456789' @@ -16,9 +18,6 @@ class TestFiles(unittest.TestCase): test_obj_path = test_coll_path + '/' + test_obj_name def setUp(self): - from irods.session import iRODSSession - import config - self.sess = iRODSSession(host=config.IRODS_SERVER_HOST, port=config.IRODS_SERVER_PORT, # 4444: why? user=config.IRODS_USER_USERNAME, diff --git a/irods/test/meta_test.py b/irods/test/meta_test.py index 7023908..84bfa05 100644 --- a/irods/test/meta_test.py +++ b/irods/test/meta_test.py @@ -5,6 +5,8 @@ from irods.meta import iRODSMeta from irods.models import (DataObject, Collection, Resource, User, DataObjectMeta, CollectionMeta, ResourceMeta, UserMeta) +from irods.session import iRODSSession +import config @@ -14,7 +16,7 @@ class TestMeta(unittest.TestCase): ''' # test data - coll_path = '/tempZone/home/rods/test_dir' + coll_path = '/{0}/home/{1}/test_dir'.format(config.IRODS_SERVER_ZONE, config.IRODS_USER_USERNAME) obj_name = 'test1' obj_path = '{0}/{1}'.format(coll_path, obj_name) @@ -24,9 +26,6 @@ class TestMeta(unittest.TestCase): def setUp(self): - from irods.session import iRODSSession - import config - self.sess = iRODSSession(host=config.IRODS_SERVER_HOST, port=config.IRODS_SERVER_PORT, user=config.IRODS_USER_USERNAME, diff --git a/irods/test/query_test.py b/irods/test/query_test.py index a9f374c..3e92a7f 100644 --- a/irods/test/query_test.py +++ b/irods/test/query_test.py @@ -3,21 +3,20 @@ import os import sys from irods.models import User, Collection, Keywords, DataObject +from irods.session import iRODSSession +import config class TestQuery(unittest.TestCase): """ """ # test data - coll_path = '/tempZone/home/rods/test_dir' + coll_path = '/{0}/home/{1}/test_dir'.format(config.IRODS_SERVER_ZONE, config.IRODS_USER_USERNAME) obj_name = 'test1' obj_path = '{0}/{1}'.format(coll_path, obj_name) def setUp(self): - from irods.session import iRODSSession - import config - self.sess = iRODSSession(host=config.IRODS_SERVER_HOST, port=config.IRODS_SERVER_PORT, # 4444 why? user=config.IRODS_USER_USERNAME, From f9b68e6c6dd44b1820c702b5c73a5c7da71d4f4f Mon Sep 17 00:00:00 2001 From: Antoine de Torcy Date: Mon, 8 Sep 2014 10:31:36 -0400 Subject: [PATCH 48/55] [#6] Test results update --- Testing.md | 16 ---------------- irods/test/Testing.md | 13 ++++++------- 2 files changed, 6 insertions(+), 23 deletions(-) delete mode 100644 Testing.md diff --git a/Testing.md b/Testing.md deleted file mode 100644 index f65ad33..0000000 --- a/Testing.md +++ /dev/null @@ -1,16 +0,0 @@ -TESTING NOTES -============= - -| Test | Status | Notes | -| ------------- |-------------|------------| -| browse_test.py | fail | `File "browse_test.py", line 12, in test_get_collection coll = sess.get_collection(path) AttributeError: 'iRODSSession' object has no attribute 'get_collection'`| -| coll_test.py | fail | requires username, password, host; fixable| -| file_test.py | fail |`File "file_test.py", line 9, in obj = sess.get_data_object("/tempZone/home/rods/test1") AttributeError: 'iRODSSession' object has no attribute 'get_data_object'`| -| message_test.py | pass || -| meta_test.py | fail | ` - File "meta_test.py", line 10, in - obj = sess.get_data_object("/tempZone/home/rods/test1") -AttributeError: 'iRODSSession' object has no attribute 'get_data_object'` | -| query_test.py | || -| test_connection.py | || - diff --git a/irods/test/Testing.md b/irods/test/Testing.md index c4eb821..9b831a3 100644 --- a/irods/test/Testing.md +++ b/irods/test/Testing.md @@ -35,11 +35,10 @@ Current Test Results | Test | Status | Notes | | ------------- |-------------|------------| -| browse_test.py | fails | `File "browse_test.py", line 12, in test_get_collection coll = sess.get_collection(path) AttributeError: 'iRODSSession' object has no attribute 'get_collection'`| -| coll_test.py | fails | requires username, password, host; fixable| -| file_test.py | fails | `File "file_test.py", line 9, in obj = sess.get_data_object("/tempZone/home/rods/test1") AttributeError: 'iRODSSession' object has no attribute 'get_data_object'`| -| message_test.py | succeeds || -| meta_test.py | fails || -| query_test.py | fails || -| test_connection.py | fails || +| collection_test.py | OK || +| connection_test.py | OK || +| file_test.py | OK || +| message_test.py | OK || +| meta_test.py | OK || +| query_test.py | OK || From dab32807076e6d717546634c813d1853bae9ebf5 Mon Sep 17 00:00:00 2001 From: Antoine de Torcy Date: Mon, 8 Sep 2014 10:38:52 -0400 Subject: [PATCH 49/55] [#6] Test results update --- irods/test/Testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/irods/test/Testing.md b/irods/test/Testing.md index 9b831a3..3ed8011 100644 --- a/irods/test/Testing.md +++ b/irods/test/Testing.md @@ -27,7 +27,7 @@ This imports all tests in the `test` directory and runs them. It will not die up Test Dependencies ----------------- -Testing appears to rely on a local irods directory. +A valid account on a running iRODS grid. See `./config.py`. Current Test Results -------------------- From 4ba5e25b8a84ec90ad3f4b85823f73aefc01a00d Mon Sep 17 00:00:00 2001 From: Antoine de Torcy Date: Wed, 10 Sep 2014 17:47:04 -0400 Subject: [PATCH 50/55] [#3] added collection metadata test --- irods/test/meta_test.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/irods/test/meta_test.py b/irods/test/meta_test.py index 84bfa05..6811fea 100644 --- a/irods/test/meta_test.py +++ b/irods/test/meta_test.py @@ -120,6 +120,32 @@ def test_remove_obj_meta(self): assert (len(meta) == 0) + def test_add_coll_meta(self): + """ + """ + + # add metadata to test collection + self.sess.metadata.add(Collection, self.coll_path, + iRODSMeta(self.attr0, self.value0)) + + # get collection metadata + meta = self.sess.metadata.get(Collection, self.coll_path) + + # assertions + assert(meta[0].name == self.attr0) + assert(meta[0].value == self.value0) + + # remove collection metadata + self.sess.metadata.remove(Collection, self.coll_path, + iRODSMeta(self.attr0, self.value0)) + + # check that metadata is gone + meta = self.sess.metadata.get(Collection, self.coll_path) + assert (len(meta) == 0) + + + + if __name__ == '__main__': # let the tests find the parent irods lib sys.path.insert(0, os.path.abspath('../..')) From 1f5511e5afdcb3e8977d6d2918cc82585c2e2251 Mon Sep 17 00:00:00 2001 From: Low Kian Seong Date: Thu, 18 Sep 2014 04:14:48 +0800 Subject: [PATCH 51/55] Update setup.py Adding more metadata to `setup.py` for a more complete listing on `pypi` --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 3c6fc99..876781a 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,8 @@ version='0.3', author='Chris LaRose', author_email='cjlarose@iplantcollaborative.org', + description='A python API for iRODS', + url='https://github.com/iPlantCollaborativeOpenSource/', packages=['irods', 'irods.message', 'irods.resource_manager'], install_requires=[ 'PrettyTable>=0.7,<1.0' From 30c84f4de82a871fa03bedd6f8ebe87a37aafb2d Mon Sep 17 00:00:00 2001 From: "J. Matt Peterson" Date: Wed, 17 Sep 2014 13:23:00 -0700 Subject: [PATCH 52/55] Change for initial pypi release. modified: .gitignore new file: AUTHORS new file: CHANGES new file: LICENSE new file: MANIFEST.in modified: setup.py --- .gitignore | 3 +++ AUTHORS | 10 ++++++++++ CHANGES | 1 + LICENSE | 13 +++++++++++++ MANIFEST.in | 1 + setup.py | 22 +++++++++++----------- 6 files changed, 39 insertions(+), 11 deletions(-) create mode 100644 AUTHORS create mode 100644 CHANGES create mode 100644 LICENSE create mode 100644 MANIFEST.in diff --git a/.gitignore b/.gitignore index 247a900..f00225b 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,6 @@ nosetests.xml Session.vim .netrwhist *~ + +# emacs +\#*\# \ No newline at end of file diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..5560158 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,10 @@ +python-irodsclient was originally created by Chris LaRose when he was a member +of the atmosphere team at the iPlant Collaborative. + +Antoine de Torcy recently provided support for iRODS 4 and integration into +the iRODS build system. + +Chris LaRose +Antoine de Torcy +J. Matt Peterson +David Knapp \ No newline at end of file diff --git a/CHANGES b/CHANGES new file mode 100644 index 0000000..83d579d --- /dev/null +++ b/CHANGES @@ -0,0 +1 @@ +0.3.1 - Initial pypi release. \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..51273ad --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2010-2014, The Arizona Board of Regents on behalf of The University of Arizona + +All rights reserved. + +Developed by: iPlant Collaborative as a collaboration between participants at BIO5 at The University of Arizona (the primary hosting institution), Cold Spring Harbor Laboratory, The University of Texas at Austin, and individual contributors. Find out more at http://www.iplantcollaborative.org/. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * Neither the name of the iPlant Collaborative, BIO5, The University of Arizona, Cold Spring Harbor Laboratory, The University of Texas at Austin, nor the names of other contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..868732e --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include AUTHORS CHANGES LICENSE README.md \ No newline at end of file diff --git a/setup.py b/setup.py index 3c6fc99..ad2abb9 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,11 @@ -from distutils.core import setup -setup( - name='python-irodsclient', - version='0.3', - author='Chris LaRose', - author_email='cjlarose@iplantcollaborative.org', - packages=['irods', 'irods.message', 'irods.resource_manager'], - install_requires=[ - 'PrettyTable>=0.7,<1.0' - ] -) +import setuptools + + +setuptools.setup(name='python-irodsclient', + version='0.3.1', + author='iPlant Collaborative', + author_email='atmodevs@gmail.com', + packages=['irods', + 'irods.message', + 'irods.resource_manager'], + install_requires=['PrettyTable>=0.7']) From dcd234c166c06bdfc40fa2ef135c2511e0a4e7ac Mon Sep 17 00:00:00 2001 From: "J. Matt Peterson" Date: Wed, 17 Sep 2014 13:37:19 -0700 Subject: [PATCH 53/55] Update version. modified: AUTHORS modified: setup.py --- AUTHORS | 3 ++- setup.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index 5560158..c5cacd5 100644 --- a/AUTHORS +++ b/AUTHORS @@ -7,4 +7,5 @@ the iRODS build system. Chris LaRose Antoine de Torcy J. Matt Peterson -David Knapp \ No newline at end of file +David Knapp +Steve "esteve" Gregory \ No newline at end of file diff --git a/setup.py b/setup.py index 6cb21d2..3c8939d 100644 --- a/setup.py +++ b/setup.py @@ -2,10 +2,10 @@ setup(name='python-irodsclient', - version='0.3.1', + version='0.4.0', author='iPlant Collaborative', author_email='atmodevs@gmail.com', description='A python API for iRODS', - url='https://github.com/iPlantCollaborativeOpenSource/', + url='https://github.com/iPlantCollaborativeOpenSource/python-irodsclient', packages=find_packages(), install_requires=['PrettyTable>=0.7.2']) From b8437272a4abf4f1c479d8ba11a98af4ff5b5584 Mon Sep 17 00:00:00 2001 From: Andrew Lenards Date: Thu, 14 Jul 2016 17:22:10 -0700 Subject: [PATCH 54/55] Added notification to see current iRODS repo --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1e143a8..ba56296 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,10 @@ -python-irodsclient +# Now maintained by iRODS at [irods/python-irodsclient](https://github.com/irods/python-irodsclient/) + +For version 0.5.0, see https://github.com/irods/python-irodsclient/ + +--- + +python-irodsclient - version 0.4.0 ============ [iRODS](https://www.irods.org) is an open-source distributed filesystem manager. This a client API implemented in python. From da4daaeee1ad460b5e65ccf9127bfcaad0766950 Mon Sep 17 00:00:00 2001 From: Andrew Lenards Date: Thu, 14 Jul 2016 17:23:47 -0700 Subject: [PATCH 55/55] Included git+pip install example --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index ba56296..5c8238f 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,11 @@ For version 0.5.0, see https://github.com/irods/python-irodsclient/ +Or, install via GitHub: +``` +pip install git+git://github.com/irods/python-irodsclient.git +``` + --- python-irodsclient - version 0.4.0