From 04db413ab09f6270ba445fd82fa4aade42a74a91 Mon Sep 17 00:00:00 2001 From: Chris Turner Date: Fri, 22 Jan 2021 18:07:41 -0600 Subject: [PATCH 1/4] add 06-distributed-computing_jobs_by_key.rst (#259) --- .../06-distributed-computing_jobs_by_key.rst | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 docs-parts/computation/06-distributed-computing_jobs_by_key.rst diff --git a/docs-parts/computation/06-distributed-computing_jobs_by_key.rst b/docs-parts/computation/06-distributed-computing_jobs_by_key.rst new file mode 100644 index 00000000..5c8a87a8 --- /dev/null +++ b/docs-parts/computation/06-distributed-computing_jobs_by_key.rst @@ -0,0 +1,36 @@ + +This can be done by using `dj.internal.hash` to convert the key as follows: + +.. code-block:: matlab + + > kh = dj.internal.hash(key); + > Lab.Jobs() & struct('key_hash', kh(1:32)) + + + ans = + + + Object Lab.Jobs + + :: the job reservation table for +Lab :: + + TABLE_NAME KEY_HASH status error_message user host pid connection_id timestamp key error_stack + _______________________ ____________________________________ ____________ _____________ ________ _________ __________ _____________ _______________________ __________ ___________ + + {'Lab.SessionAnalysis'} {'jA9sN_5PvusWwmznLGcAZbTn5pGtba-z'} {'error'} {0×0 char} {'datajoint@localhost'} {'localhost'} 6.5356e+05 1919 {'2021-01-22 23:50:07'} {'=BLOB='} {'=BLOB='} + + 1 tuples (0.127 s) + + > del(Lab.Jobs() & struct('key_hash', kh(1:32))); + + > Lab.Jobs() & struct('key_hash', kh(1:32)) + + ans = + + + Object Lab.Jobs + + :: the job reservation table for +Lab :: + + 0 tuples (0.0309 s) + From 31d8a415ff3bdde5639cfd402f673f664cef68a1 Mon Sep 17 00:00:00 2001 From: Chris Turner Date: Thu, 18 Feb 2021 17:33:55 -0600 Subject: [PATCH 2/4] add dj.key_hash() for job key mgmt, adjust code/tests/examples --- +dj/+internal/AutoPopulate.m | 4 ++-- +dj/key_hash.m | 7 +++++++ .../computation/06-distributed-computing_jobs_by_key.rst | 8 ++++---- tests/test_schemas/+Lab/SessionAnalysis.m | 9 +++++++++ 4 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 +dj/key_hash.m diff --git a/+dj/+internal/AutoPopulate.m b/+dj/+internal/AutoPopulate.m index fd3f5bf0..b6f63c56 100755 --- a/+dj/+internal/AutoPopulate.m +++ b/+dj/+internal/AutoPopulate.m @@ -349,8 +349,8 @@ function cleanup(self, key) function jobKey = makeJobKey(self, key) - hash = dj.internal.hash(key); - jobKey = struct('table_name', self.className, 'key_hash', hash(1:32)); + jobKey = struct('table_name', self.className, ... + 'key_hash', key_hash(key)); end diff --git a/+dj/key_hash.m b/+dj/key_hash.m new file mode 100644 index 00000000..1f152d41 --- /dev/null +++ b/+dj/key_hash.m @@ -0,0 +1,7 @@ +% dj.key_hash - key hashing function for populate jobs, etc + +function s = key_hash(key) + s = dj.internal.hash(key); + s = s(1:32); +end + diff --git a/docs-parts/computation/06-distributed-computing_jobs_by_key.rst b/docs-parts/computation/06-distributed-computing_jobs_by_key.rst index 5c8a87a8..59b5f548 100644 --- a/docs-parts/computation/06-distributed-computing_jobs_by_key.rst +++ b/docs-parts/computation/06-distributed-computing_jobs_by_key.rst @@ -3,8 +3,8 @@ This can be done by using `dj.internal.hash` to convert the key as follows: .. code-block:: matlab - > kh = dj.internal.hash(key); - > Lab.Jobs() & struct('key_hash', kh(1:32)) + > job_key = struct('key_hash', dj.key_hash(key)); + > Lab.Jobs() & job_key ans = @@ -21,9 +21,9 @@ This can be done by using `dj.internal.hash` to convert the key as follows: 1 tuples (0.127 s) - > del(Lab.Jobs() & struct('key_hash', kh(1:32))); + > del(Lab.Jobs() & job_key; - > Lab.Jobs() & struct('key_hash', kh(1:32)) + > Lab.Jobs() & job_key ans = diff --git a/tests/test_schemas/+Lab/SessionAnalysis.m b/tests/test_schemas/+Lab/SessionAnalysis.m index 258f35fa..64a68d28 100644 --- a/tests/test_schemas/+Lab/SessionAnalysis.m +++ b/tests/test_schemas/+Lab/SessionAnalysis.m @@ -12,6 +12,15 @@ function makeTuples(self,key) r = sprintf('connection_id = %d', c.serverId); j = fetch(Lab.Jobs() & r, '*'); + + jobkey = struct('key_hash', dj.key_hash(key)); + fprintf('before') + Lab.Jobs() & jobkey + + del(Lab.Jobs() & jobkey + + fprintf('after') + Lab.Jobs() & jobkey if isempty(j) key.session_analysis = key.session_id; From 1c88600dfd51defc9aa2cd2b4c9c96054386b72a Mon Sep 17 00:00:00 2001 From: Chris Turner Date: Thu, 18 Feb 2021 17:45:22 -0600 Subject: [PATCH 3/4] tests/test_schemas/+Lab/SessionAnalysis.m: back out debug code commit --- tests/test_schemas/+Lab/SessionAnalysis.m | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/test_schemas/+Lab/SessionAnalysis.m b/tests/test_schemas/+Lab/SessionAnalysis.m index 64a68d28..ed386c42 100644 --- a/tests/test_schemas/+Lab/SessionAnalysis.m +++ b/tests/test_schemas/+Lab/SessionAnalysis.m @@ -13,15 +13,6 @@ function makeTuples(self,key) j = fetch(Lab.Jobs() & r, '*'); - jobkey = struct('key_hash', dj.key_hash(key)); - fprintf('before') - Lab.Jobs() & jobkey - - del(Lab.Jobs() & jobkey - - fprintf('after') - Lab.Jobs() & jobkey - if isempty(j) key.session_analysis = key.session_id; else From 24632716666a7544cec767f5fbc9d2e5e566bbbb Mon Sep 17 00:00:00 2001 From: Chris Turner Date: Thu, 18 Feb 2021 18:22:20 -0600 Subject: [PATCH 4/4] add table name to 06-distributed-computing_jobs_by_key.rst example --- .../computation/06-distributed-computing_jobs_by_key.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs-parts/computation/06-distributed-computing_jobs_by_key.rst b/docs-parts/computation/06-distributed-computing_jobs_by_key.rst index 59b5f548..9240bbe9 100644 --- a/docs-parts/computation/06-distributed-computing_jobs_by_key.rst +++ b/docs-parts/computation/06-distributed-computing_jobs_by_key.rst @@ -3,7 +3,8 @@ This can be done by using `dj.internal.hash` to convert the key as follows: .. code-block:: matlab - > job_key = struct('key_hash', dj.key_hash(key)); + > job_key = struct('table_name', 'Lab.SessionAnalysis', ... + 'key_hash', dj.key_hash(key)); > Lab.Jobs() & job_key