From 0a1c670577c2bb274b0d429dcdfb96a45ecbae8a Mon Sep 17 00:00:00 2001 From: Stuart Pullinger Date: Wed, 12 Mar 2014 15:17:13 +0000 Subject: [PATCH 01/45] Added function to delete duplicate records after data is migrated from EMI2 database --- scripts/migrate_apel.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/scripts/migrate_apel.py b/scripts/migrate_apel.py index ee1a72d8d..a5846d7a7 100755 --- a/scripts/migrate_apel.py +++ b/scripts/migrate_apel.py @@ -57,6 +57,21 @@ CALLPROC_STMT = """CALL InsertJobRecord(%s, %s, %s, 'None', %s, %s, %s, %s, %s, %s, %s, %s, %s, NULL, NULL, %s, %s, %s, %s, %s, %s, %s, %s, %s)""" +DUPLICATES_JOIN = """ FROM JobRecords AS t + LEFT JOIN MachineNames as m + on (t.MachineNameID = m.id) + INNER JOIN (SELECT LocalJobId, + EndTime + FROM JobRecords + LEFT JOIN MachineNames + on (JobRecords.MachineNameID = MachineNames.id) + WHERE MachineNames.name = 'MachineName' ) + AS u + ON (m.name = 'MachineName' AND t.LocalJobId = u.LocalJobId AND t.EndTime = u.EndTime); """ + +COUNT_DUPLICATES_STMT = "SELECT count(*) " + DUPLICATES_JOIN + +DELETE_DUPLICATES_STMT = "DELETE t " + DUPLICATES_JOIN def parse_timestamp(string, format="%Y-%m-%dT%H:%M:%SZ"): ''' @@ -187,6 +202,23 @@ def delete_old_records(db, cutoff): c.execute(REMOVE_PROC) db.commit() +def delete_duplicates(db): + ''' + Delete all records that have been migrated but are duplicates of records + that are in the database already + ''' + c = db.cursor() + + c.execute(COUNT_DUPLICATES_STMT) + num_duplicates = c.fetchone()[0] + sys.stdout.write('Found %d duplicate entries\n' % num_duplicates) + + sys.stdout.write('Deleting duplicates\n') + c.execute(DELETE_DUPLICATES_STMT) + num_deleted = c.fetchone()[0] + sys.stdout.write('Deleted %d duplicate entries\n' % num_deleted) + + db.commit() def main(): ''' @@ -234,6 +266,9 @@ def main(): sys.stdout.write("\n Deleting all records older than %s\n" % cutoff) sys.stdout.write(" from %s:%s ...\n" % (host2, dbname2)) delete_old_records(db2, cutoff) + + # delete duplicates + delete_duplicates(db2) sys.stdout.write(" Complete.\n\n\n") From bf7e01cfda9e7dc62af23ef1e5526c978b269e9f Mon Sep 17 00:00:00 2001 From: Stuart Pullinger Date: Wed, 12 Mar 2014 15:34:51 +0000 Subject: [PATCH 02/45] Changed call after database delete to 'SELECT ROW_COUNT()' to extract the number of records deleted. --- scripts/migrate_apel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/migrate_apel.py b/scripts/migrate_apel.py index a5846d7a7..7c1cc584f 100755 --- a/scripts/migrate_apel.py +++ b/scripts/migrate_apel.py @@ -215,7 +215,7 @@ def delete_duplicates(db): sys.stdout.write('Deleting duplicates\n') c.execute(DELETE_DUPLICATES_STMT) - num_deleted = c.fetchone()[0] + num_deleted = c.execute('SELECT ROW_COUNT()')[0] sys.stdout.write('Deleted %d duplicate entries\n' % num_deleted) db.commit() From 22bea7256452597ee9b16edccda2f573ab744cf8 Mon Sep 17 00:00:00 2001 From: Stuart Pullinger Date: Wed, 12 Mar 2014 16:00:01 +0000 Subject: [PATCH 03/45] Changed again to correctly get the number of modified rows from the delete statement. --- scripts/migrate_apel.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/migrate_apel.py b/scripts/migrate_apel.py index 7c1cc584f..a63b90b20 100755 --- a/scripts/migrate_apel.py +++ b/scripts/migrate_apel.py @@ -211,12 +211,12 @@ def delete_duplicates(db): c.execute(COUNT_DUPLICATES_STMT) num_duplicates = c.fetchone()[0] - sys.stdout.write('Found %d duplicate entries\n' % num_duplicates) + sys.stdout.write(' Found %d duplicate entries\n' % num_duplicates) - sys.stdout.write('Deleting duplicates\n') + sys.stdout.write(' Deleting duplicates\n') c.execute(DELETE_DUPLICATES_STMT) - num_deleted = c.execute('SELECT ROW_COUNT()')[0] - sys.stdout.write('Deleted %d duplicate entries\n' % num_deleted) + num_deleted = db.affected_rows() + sys.stdout.write(' Deleted %d duplicate entries\n' % num_deleted) db.commit() From c34948374be0b8ffa9b0c3bdad0eb66c1b8d275b Mon Sep 17 00:00:00 2001 From: Stuart Pullinger Date: Wed, 12 Mar 2014 16:04:49 +0000 Subject: [PATCH 04/45] Added some newlines so the output is prettier. --- scripts/migrate_apel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/migrate_apel.py b/scripts/migrate_apel.py index a63b90b20..eac1744db 100755 --- a/scripts/migrate_apel.py +++ b/scripts/migrate_apel.py @@ -216,7 +216,7 @@ def delete_duplicates(db): sys.stdout.write(' Deleting duplicates\n') c.execute(DELETE_DUPLICATES_STMT) num_deleted = db.affected_rows() - sys.stdout.write(' Deleted %d duplicate entries\n' % num_deleted) + sys.stdout.write(' Deleted %d duplicate entries\n\n\n' % num_deleted) db.commit() @@ -266,6 +266,7 @@ def main(): sys.stdout.write("\n Deleting all records older than %s\n" % cutoff) sys.stdout.write(" from %s:%s ...\n" % (host2, dbname2)) delete_old_records(db2, cutoff) + sys.stdout.write(" Done.\n\n\n") # delete duplicates delete_duplicates(db2) From cef242e2888659278a197693729ae712449b2ef1 Mon Sep 17 00:00:00 2001 From: stuartpullinger Date: Sun, 18 May 2014 11:30:09 +0100 Subject: [PATCH 05/45] Corrected join from = to != --- scripts/migrate_apel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/migrate_apel.py b/scripts/migrate_apel.py index eac1744db..f6011b3d4 100755 --- a/scripts/migrate_apel.py +++ b/scripts/migrate_apel.py @@ -67,7 +67,7 @@ on (JobRecords.MachineNameID = MachineNames.id) WHERE MachineNames.name = 'MachineName' ) AS u - ON (m.name = 'MachineName' AND t.LocalJobId = u.LocalJobId AND t.EndTime = u.EndTime); """ + ON (m.name != 'MachineName' AND t.LocalJobId = u.LocalJobId AND t.EndTime = u.EndTime); """ COUNT_DUPLICATES_STMT = "SELECT count(*) " + DUPLICATES_JOIN From a2e416521deb04c5dd5271c45ae934cfca252984 Mon Sep 17 00:00:00 2001 From: stuartpullinger Date: Sun, 18 May 2014 11:32:32 +0100 Subject: [PATCH 06/45] Corrected join again, putting != in the right place --- scripts/migrate_apel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/migrate_apel.py b/scripts/migrate_apel.py index f6011b3d4..4d1822204 100755 --- a/scripts/migrate_apel.py +++ b/scripts/migrate_apel.py @@ -65,9 +65,9 @@ FROM JobRecords LEFT JOIN MachineNames on (JobRecords.MachineNameID = MachineNames.id) - WHERE MachineNames.name = 'MachineName' ) + WHERE MachineNames.name != 'MachineName' ) AS u - ON (m.name != 'MachineName' AND t.LocalJobId = u.LocalJobId AND t.EndTime = u.EndTime); """ + ON (m.name = 'MachineName' AND t.LocalJobId = u.LocalJobId AND t.EndTime = u.EndTime); """ COUNT_DUPLICATES_STMT = "SELECT count(*) " + DUPLICATES_JOIN From 8ea3809502258f6a76b16e82dbefca004369198b Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Tue, 7 Oct 2014 15:53:02 +0400 Subject: [PATCH 07/45] Sort log files to be processed It is much easier to skim over logs and search for some problems if you know that files are processed in some particular order. Plain lexicographical sort is fine for all files that have YYYYMMDD as the non-constant part of the name: it coincides with sort by date that is the most natural thing. Signed-off-by: Adrian Coveney --- bin/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/parser.py b/bin/parser.py index fd4ba63f4..261932d15 100644 --- a/bin/parser.py +++ b/bin/parser.py @@ -153,7 +153,7 @@ def scan_dir(parser, dirpath, reparse, expr, apel_db, processed): try: log.info('Scanning directory: %s' % dirpath) - for item in os.listdir(dirpath): + for item in sorted(os.listdir(dirpath)): abs_file = os.path.join(dirpath, item) if os.path.isfile(abs_file) and expr.match(item): # first, calculate the hash of the file: From 570595e266e190a9fe0959adc70b41ca64975740 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 19 Mar 2015 17:16:12 +0000 Subject: [PATCH 08/45] Activate Travis containerization --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index bf541dcc1..99c09e04a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,8 @@ matrix: - python: "3.4" fast_finish: true +sudo: false + # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors install: - pip install MySQL-python From 0ae44ca4b4d1d86831ff313525aa9e51005f9b1a Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 20 Mar 2015 08:47:59 +0000 Subject: [PATCH 09/45] Add caching of pip during build --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 99c09e04a..0a4fb846f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ matrix: fast_finish: true sudo: false +cache: pip # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors install: From 965949daa33e274265ebd63dcefdbe33ac0a57af Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 20 Mar 2015 13:32:47 +0000 Subject: [PATCH 10/45] Move pip install requirements to requirements.txt - Move pip install requirements to requirements.txt and remove pip install list so that Travis runs the default pip install command so that the pip cache works. --- .travis.yml | 9 --------- requirements.txt | 7 +++++++ 2 files changed, 7 insertions(+), 9 deletions(-) create mode 100644 requirements.txt diff --git a/.travis.yml b/.travis.yml index 0a4fb846f..9b9517ae6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,15 +12,6 @@ matrix: sudo: false cache: pip -# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors -install: - - pip install MySQL-python - - pip install python-ldap - - pip install iso8601 - - pip install dirq - - pip install unittest2 - - pip install coveralls - # command to run tests, e.g. python setup.py test script: - export PYTHONPATH=$PYTHONPATH:`pwd -P` diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..4ed91cae9 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +MySQL-python +python-ldap +iso8601 +dirq +# Dependencies for testing +unittest2 +coveralls \ No newline at end of file From 24019943067f61ac926c717be2d84fbb537b98e8 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 20 Mar 2015 13:49:15 +0000 Subject: [PATCH 11/45] Tidy up Travis file - Move commands that aren't actually tests to before_script section so that they don't fail the build if they fail, they will error it instead. - Add comments. --- .travis.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9b9517ae6..616863b80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,13 +9,20 @@ matrix: - python: "3.4" fast_finish: true +# Route build to container-based infrastructure sudo: false +# Cache the dependencies installed by pip cache: pip -# command to run tests, e.g. python setup.py test -script: +# Install defaults to "pip install -r requirements.txt" + +# Commands that prepare things for the test +before_script: - export PYTHONPATH=$PYTHONPATH:`pwd -P` - cd test + +# Command to run tests +script: - coverage run --source=apel,bin -m unittest2 discover after_success: From b89f1d471eb5ea98ff4cc90214fad2464c1343d7 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 20 Mar 2015 14:25:09 +0000 Subject: [PATCH 12/45] Add buffer option to unittest invocation --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 616863b80..7a45a7023 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ before_script: # Command to run tests script: - - coverage run --source=apel,bin -m unittest2 discover + - coverage run --source=apel,bin -m unittest2 discover --buffer after_success: - coveralls From 75306f5a8938584287879e21761cc5e47df36830 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 4 Jun 2015 16:15:41 +0100 Subject: [PATCH 13/45] Change job count in normalised super summary view - Change the VNormalisedSuperSummary view to use the sum of NumberOfJobs as it does not use the same grouping as the index of the underlying HybridSuperSummaries table and so was displaying low values for NumberOfJobs. --- schemas/server.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/server.sql b/schemas/server.sql index 907ccad75..d712c81f3 100644 --- a/schemas/server.sql +++ b/schemas/server.sql @@ -768,7 +768,7 @@ CREATE VIEW VNormalisedSuperSummaries AS SUM(CpuDuration) AS CpuDuration, SUM(NormalisedWallDuration) AS NormalisedWallDuration, SUM(NormalisedCpuDuration) AS NormalisedCpuDuration, - NumberOfJobs + SUM(NumberOfJobs) AS NumberOfJobs FROM HybridSuperSummaries, Sites AS site, DNs AS userdn, From 55ce188184a8efbeda5bb8cab3f68836f89c8db4 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 4 Jun 2015 17:17:39 +0100 Subject: [PATCH 14/45] Add MIN and MAX to aggregated EndTime fields - Add MIN and MAX to aggregated EndTime fields in VNormalisedSuperSummaies as this view aggregates the records more than VSuperSummaries. --- schemas/server.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schemas/server.sql b/schemas/server.sql index d712c81f3..dd8d961f0 100644 --- a/schemas/server.sql +++ b/schemas/server.sql @@ -762,8 +762,8 @@ CREATE VIEW VNormalisedSuperSummaries AS Infrastructure, NodeCount, Processors, - EarliestEndTime, - LatestEndTime, + MIN(EarliestEndTime) AS EarliestEndTime, + MAX(LatestEndTime) AS LatestEndTime, SUM(WallDuration) AS WallDuration, SUM(CpuDuration) AS CpuDuration, SUM(NormalisedWallDuration) AS NormalisedWallDuration, From 1af3e67f88b9a155aff4a199a00dd13a1e659105 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 24 Feb 2015 15:18:34 +0000 Subject: [PATCH 15/45] Tidy up Travis options - Move some commands to before_script section so that if they fail the build will be marked as errored and not failed as they are set-up commands rahter than test commands. - Remove unnecessary comments (refer to Travis docs) and added comment to highlight the dependencies which are needed for the tests themselves. --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7a45a7023..ccbdff081 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,8 +22,6 @@ before_script: - cd test # Command to run tests -script: - - coverage run --source=apel,bin -m unittest2 discover --buffer +script: coverage run --source=apel,bin -m unittest2 discover -after_success: - - coveralls +after_success: coveralls From 08c63a74bee3a1087ca416b7e8364f3706ac6300 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 24 Feb 2015 16:31:34 +0000 Subject: [PATCH 16/45] Add basic MySQL database test --- .travis.yml | 1 + test/test_mysql.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 test/test_mysql.py diff --git a/.travis.yml b/.travis.yml index ccbdff081..a0c7e4708 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,7 @@ cache: pip # Commands that prepare things for the test before_script: + - mysql -e 'CREATE DATABASE apel_unittest;' - export PYTHONPATH=$PYTHONPATH:`pwd -P` - cd test diff --git a/test/test_mysql.py b/test/test_mysql.py new file mode 100644 index 000000000..d8d5adf96 --- /dev/null +++ b/test/test_mysql.py @@ -0,0 +1,25 @@ +import os +import subprocess +import unittest + +import apel.db.apeldb + + +class MysqlTest(unittest.TestCase): + # These test cases require a local MySQL db server with a apel_unittest db + def setUp(self): + schema_path = os.path.abspath(os.path.join('..', 'schemas', + 'server.sql')) + schema_handle = open(schema_path) + subprocess.Popen(['mysql', 'apel_unittest'], stdin=schema_handle).wait() + schema_handle.close() + + self.db = apel.db.apeldb.ApelDb('mysql', 'localhost', 3306, 'root', '', + 'apel_unittest') + + def test(self): + self.db.test_connection() + + +if __name__ == '__main__': + unittest.main() From ba142c27cb4c1e5370d4869c9a8563809f53143a Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 25 Feb 2015 16:01:10 +0000 Subject: [PATCH 17/45] Add more database testing - Add cx_Oracle to Travis CI install section so that more code is covered. - Add basic test for loading and getting a simple job record. --- test/test_mysql.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/test/test_mysql.py b/test/test_mysql.py index d8d5adf96..7927171dc 100644 --- a/test/test_mysql.py +++ b/test/test_mysql.py @@ -1,8 +1,10 @@ +import datetime import os import subprocess import unittest import apel.db.apeldb +import apel.db.records.job class MysqlTest(unittest.TestCase): @@ -17,9 +19,30 @@ def setUp(self): self.db = apel.db.apeldb.ApelDb('mysql', 'localhost', 3306, 'root', '', 'apel_unittest') - def test(self): + def test_connection(self): self.db.test_connection() + def test_load_and_get(self): + job = apel.db.records.job.JobRecord() + job._record_content = {'Site': 'testSite', 'LocalJobId': 'testJob', + 'SubmitHost': 'testHost', + 'WallDuration': 10, 'CpuDuration': 10, + 'StartTime': datetime.datetime.fromtimestamp(123456), + 'EndTime': datetime.datetime.fromtimestamp(654321), + 'ServiceLevelType': 'HEPSPEC', + 'ServiceLevel': 3} + items_in = job._record_content.items() + record_list = [job] + # load_records changes the 'job' job record as it calls _check_fields + # which adds placeholders to empty fields + self.db.load_records(record_list, source='testDN') + + records_out = self.db.get_records(apel.db.records.job.JobRecord) + items_out = list(records_out)[0][0]._record_content.items() + # Check that items_in is a subset of items_out + # Can't use 'all()' rather than comparing the length as Python 2.4 + self.assertEqual([item in items_out for item in items_in].count(True), len(items_in)) + if __name__ == '__main__': unittest.main() From 381f08b00fcf08851446f62d3a38faa5ae50bc62 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 26 Feb 2015 12:14:00 +0000 Subject: [PATCH 18/45] Add more db tests and change name of one --- test/test_mysql.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/test/test_mysql.py b/test/test_mysql.py index 7927171dc..ba4574554 100644 --- a/test/test_mysql.py +++ b/test/test_mysql.py @@ -19,9 +19,32 @@ def setUp(self): self.db = apel.db.apeldb.ApelDb('mysql', 'localhost', 3306, 'root', '', 'apel_unittest') - def test_connection(self): + def test_test_connection(self): + """Basic check that test_connection works without error.""" self.db.test_connection() + def test_bad_connection(self): + """Check that initialising ApelDb fails if a bad password is used.""" + self.assertRaises(apel.db.apeldb.ApelDbException, apel.db.apeldb.ApelDb, + 'mysql', 'localhost', 3306, 'root', 'badpassword', + 'apel_badtest') + + def test_lost_connection(self): + """ + Check that a lost connection to the db raises an exception. + + Simulate the lost connection by changing the port. + """ + self.db._db_port = 1234 + self.assertRaises(apel.db.apeldb.ApelDbException, + self.db.test_connection) + + def test_bad_loads(self): + """Check that empty loads return None and bad types raise exception.""" + self.assertTrue(self.db.load_records([], source='testDN') is None) + self.assertRaises(apel.db.apeldb.ApelDbException, + self.db.load_records, [1234], source='testDN') + def test_load_and_get(self): job = apel.db.records.job.JobRecord() job._record_content = {'Site': 'testSite', 'LocalJobId': 'testJob', From dcd267ad6d92bdeb2214941b0f9efbf53459cbe9 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 26 Feb 2015 12:20:49 +0000 Subject: [PATCH 19/45] Change port to host for lost connection test - Change port to host for lost connection test as Travis CI seems to bind MySQL to all ports on localhost. --- test/test_mysql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_mysql.py b/test/test_mysql.py index ba4574554..fa880f00a 100644 --- a/test/test_mysql.py +++ b/test/test_mysql.py @@ -33,9 +33,9 @@ def test_lost_connection(self): """ Check that a lost connection to the db raises an exception. - Simulate the lost connection by changing the port. + Simulate the lost connection by changing the host. """ - self.db._db_port = 1234 + self.db._db_host = 'badhost' self.assertRaises(apel.db.apeldb.ApelDbException, self.db.test_connection) From cfaf92ef07be5836f8fc6e5d60d2f2457a39e155 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 9 Jun 2015 17:07:11 +0100 Subject: [PATCH 20/45] Add test for LastUpdated db table --- test/test_mysql.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/test_mysql.py b/test/test_mysql.py index fa880f00a..a8e06d192 100644 --- a/test/test_mysql.py +++ b/test/test_mysql.py @@ -8,7 +8,7 @@ class MysqlTest(unittest.TestCase): - # These test cases require a local MySQL db server with a apel_unittest db + # These test cases require a local MySQL db server with an apel_unittest db def setUp(self): schema_path = os.path.abspath(os.path.join('..', 'schemas', 'server.sql')) @@ -66,6 +66,17 @@ def test_load_and_get(self): # Can't use 'all()' rather than comparing the length as Python 2.4 self.assertEqual([item in items_out for item in items_in].count(True), len(items_in)) + def test_last_update(self): + """ + Check that the LastUpdated table can be set and queried. + + It should not be set initially, so should return None, then should + return a time after being set. + """ + self.assertTrue(self.db.get_last_updated() is None) + self.assertTrue(self.db.set_updated()) + self.assertTrue(type(self.db.get_last_updated()) is datetime.datetime) + if __name__ == '__main__': unittest.main() From 7e4ebd9c937207fe3b54947dc67f6f8e6c7ff3b3 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 11 Jun 2015 17:16:19 +0100 Subject: [PATCH 21/45] Add SQL validity test --- test/test_schemas.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 test/test_schemas.py diff --git a/test/test_schemas.py b/test/test_schemas.py new file mode 100644 index 000000000..e32a16409 --- /dev/null +++ b/test/test_schemas.py @@ -0,0 +1,37 @@ +import os +from subprocess import Popen, PIPE +import unittest + + +class SchemaTest(unittest.TestCase): + # These test cases require a local MySQL db server + def setUp(self): + Popen(['mysql', "-e 'DROP DATABASE IF EXISTS apel_unittest; CREATE DATABASE apel_unittest;'"]) + + def tearDown(self): + Popen(['mysql', "-e 'DROP DATABASE apel_unittest;'"]).wait() + + +def make_schema_test(schema): + def schema_test(self): + if schema == 'server-extra': + schema_path = os.path.abspath(os.path.join('..', 'schemas', 'server.sql')) + schema_handle = open(schema_path) + Popen(['mysql', 'apel_unittest'], stdin=schema_handle).wait() + schema_handle.close() + schema_path = os.path.abspath(os.path.join('..', 'schemas', schema + '.sql')) + schema_handle = open(schema_path) + p = Popen(['mysql', 'apel_unittest'], stdin=schema_handle, stderr=PIPE) + p.wait() + schema_handle.close() + self.assertFalse(p.returncode, p.communicate()[1]) + return schema_test + +for schema in ('client', 'cloud', 'server', 'server-extra', 'storage'): + test_method = make_schema_test(schema) + test_method.__name__ = 'test_%s_schema' % schema + setattr(SchemaTest, test_method.__name__, test_method) + + +if __name__ == '__main__': + unittest.main() From e5e42dbae037b90547e50e56caf2ea671e93adbf Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 12 Jun 2015 11:33:23 +0100 Subject: [PATCH 22/45] Fix db queries --- test/test_schemas.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/test/test_schemas.py b/test/test_schemas.py index e32a16409..45167ab44 100644 --- a/test/test_schemas.py +++ b/test/test_schemas.py @@ -1,32 +1,39 @@ import os -from subprocess import Popen, PIPE +from subprocess import call, Popen, PIPE import unittest class SchemaTest(unittest.TestCase): # These test cases require a local MySQL db server def setUp(self): - Popen(['mysql', "-e 'DROP DATABASE IF EXISTS apel_unittest; CREATE DATABASE apel_unittest;'"]) + query = ('DROP DATABASE IF EXISTS apel_unittest;' + 'CREATE DATABASE apel_unittest;') + call(['mysql', '-e', query]) def tearDown(self): - Popen(['mysql', "-e 'DROP DATABASE apel_unittest;'"]).wait() + call(['mysql', '-e', 'DROP DATABASE apel_unittest;']) def make_schema_test(schema): + """Make a test case that will check the given schema.""" def schema_test(self): if schema == 'server-extra': - schema_path = os.path.abspath(os.path.join('..', 'schemas', 'server.sql')) - schema_handle = open(schema_path) - Popen(['mysql', 'apel_unittest'], stdin=schema_handle).wait() - schema_handle.close() - schema_path = os.path.abspath(os.path.join('..', 'schemas', schema + '.sql')) + # server-extra.sql needs server.sql loaded first + parent_schema_path = os.path.abspath(os.path.join('..', 'schemas', + 'server.sql')) + parent_schema_handle = open(parent_schema_path) + call(['mysql', 'apel_unittest'], stdin=parent_schema_handle) + parent_schema_handle.close() + schema_path = os.path.abspath(os.path.join('..', 'schemas', + '%s.sql' % schema)) schema_handle = open(schema_path) p = Popen(['mysql', 'apel_unittest'], stdin=schema_handle, stderr=PIPE) - p.wait() schema_handle.close() + p.wait() self.assertFalse(p.returncode, p.communicate()[1]) return schema_test + for schema in ('client', 'cloud', 'server', 'server-extra', 'storage'): test_method = make_schema_test(schema) test_method.__name__ = 'test_%s_schema' % schema From 9163775cdb56428a8cdb14c3ec0fe9fcd24f1742 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 12 Jun 2015 11:50:55 +0100 Subject: [PATCH 23/45] Add automatic schema discovery to schema test --- test/test_schemas.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_schemas.py b/test/test_schemas.py index 45167ab44..1c347bf5a 100644 --- a/test/test_schemas.py +++ b/test/test_schemas.py @@ -17,15 +17,14 @@ def tearDown(self): def make_schema_test(schema): """Make a test case that will check the given schema.""" def schema_test(self): - if schema == 'server-extra': + if schema == 'server-extra.sql': # server-extra.sql needs server.sql loaded first parent_schema_path = os.path.abspath(os.path.join('..', 'schemas', 'server.sql')) parent_schema_handle = open(parent_schema_path) call(['mysql', 'apel_unittest'], stdin=parent_schema_handle) parent_schema_handle.close() - schema_path = os.path.abspath(os.path.join('..', 'schemas', - '%s.sql' % schema)) + schema_path = os.path.abspath(os.path.join('..', 'schemas', schema)) schema_handle = open(schema_path) p = Popen(['mysql', 'apel_unittest'], stdin=schema_handle, stderr=PIPE) schema_handle.close() @@ -34,9 +33,10 @@ def schema_test(self): return schema_test -for schema in ('client', 'cloud', 'server', 'server-extra', 'storage'): +# Make a test case for each schema found in the schemas directory +for schema in os.listdir(os.path.abspath(os.path.join('..', 'schemas'))): test_method = make_schema_test(schema) - test_method.__name__ = 'test_%s_schema' % schema + test_method.__name__ = 'test_%s_schema' % schema[:-4] setattr(SchemaTest, test_method.__name__, test_method) From 97ea9aede91ba8c5b39701d226b92b6037af44da Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 12 Jun 2015 11:58:48 +0100 Subject: [PATCH 24/45] Add db setup and teardown to mysql test fixture --- test/test_mysql.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/test_mysql.py b/test/test_mysql.py index a8e06d192..d1005d8aa 100644 --- a/test/test_mysql.py +++ b/test/test_mysql.py @@ -8,17 +8,24 @@ class MysqlTest(unittest.TestCase): - # These test cases require a local MySQL db server with an apel_unittest db + # These test cases require a local MySQL db server def setUp(self): + query = ('DROP DATABASE IF EXISTS apel_unittest;' + 'CREATE DATABASE apel_unittest;') + subprocess.call(['mysql', '-e', query]) + schema_path = os.path.abspath(os.path.join('..', 'schemas', 'server.sql')) schema_handle = open(schema_path) - subprocess.Popen(['mysql', 'apel_unittest'], stdin=schema_handle).wait() + subprocess.call(['mysql', 'apel_unittest'], stdin=schema_handle) schema_handle.close() self.db = apel.db.apeldb.ApelDb('mysql', 'localhost', 3306, 'root', '', 'apel_unittest') + def tearDown(self): + subprocess.call(['mysql', '-e', 'DROP DATABASE apel_unittest;']) + def test_test_connection(self): """Basic check that test_connection works without error.""" self.db.test_connection() From 83ec8ae126d3b1b4978540836c9bb0a36fc8e968 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 12 Jun 2015 12:52:37 +0100 Subject: [PATCH 25/45] Remove database setup from Travis file - Remove database setup from Travis file as it is now done per test in the Python files. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a0c7e4708..ccbdff081 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,6 @@ cache: pip # Commands that prepare things for the test before_script: - - mysql -e 'CREATE DATABASE apel_unittest;' - export PYTHONPATH=$PYTHONPATH:`pwd -P` - cd test From 7f0572f56663fc09b3401f25dc3f574cc2738b36 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 12 Jun 2015 13:54:10 +0100 Subject: [PATCH 26/45] Make db tests compatible with Windows This is set up to work with a default install of MySQL Server 5.1 but can be changed easily for local testing. --- test/test_mysql.py | 12 ++++++++---- test/test_schemas.py | 14 +++++++++----- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/test/test_mysql.py b/test/test_mysql.py index d1005d8aa..54ac7cc78 100644 --- a/test/test_mysql.py +++ b/test/test_mysql.py @@ -7,24 +7,28 @@ import apel.db.records.job +if os.name == 'nt': + os.environ['PATH'] += ';C:/Program Files/MySQL/MySQL Server 5.1/bin/' + + class MysqlTest(unittest.TestCase): - # These test cases require a local MySQL db server + # These test cases require a local MySQL db server with no password on root def setUp(self): query = ('DROP DATABASE IF EXISTS apel_unittest;' 'CREATE DATABASE apel_unittest;') - subprocess.call(['mysql', '-e', query]) + subprocess.call(['mysql', '-u', 'root', '-e', query]) schema_path = os.path.abspath(os.path.join('..', 'schemas', 'server.sql')) schema_handle = open(schema_path) - subprocess.call(['mysql', 'apel_unittest'], stdin=schema_handle) + subprocess.call(['mysql', '-u', 'root', 'apel_unittest'], stdin=schema_handle) schema_handle.close() self.db = apel.db.apeldb.ApelDb('mysql', 'localhost', 3306, 'root', '', 'apel_unittest') def tearDown(self): - subprocess.call(['mysql', '-e', 'DROP DATABASE apel_unittest;']) + subprocess.call(['mysql', '-u', 'root', '-e', 'DROP DATABASE apel_unittest;']) def test_test_connection(self): """Basic check that test_connection works without error.""" diff --git a/test/test_schemas.py b/test/test_schemas.py index 1c347bf5a..17ace05df 100644 --- a/test/test_schemas.py +++ b/test/test_schemas.py @@ -3,15 +3,19 @@ import unittest +if os.name == 'nt': + os.environ['PATH'] += ';C:/Program Files/MySQL/MySQL Server 5.1/bin/' + + class SchemaTest(unittest.TestCase): - # These test cases require a local MySQL db server + # These test cases require a local MySQL db server with no password on root def setUp(self): query = ('DROP DATABASE IF EXISTS apel_unittest;' 'CREATE DATABASE apel_unittest;') - call(['mysql', '-e', query]) + call(['mysql', '-u', 'root', '-e', query]) def tearDown(self): - call(['mysql', '-e', 'DROP DATABASE apel_unittest;']) + call(['mysql', '-u', 'root', '-e', 'DROP DATABASE apel_unittest;']) def make_schema_test(schema): @@ -22,11 +26,11 @@ def schema_test(self): parent_schema_path = os.path.abspath(os.path.join('..', 'schemas', 'server.sql')) parent_schema_handle = open(parent_schema_path) - call(['mysql', 'apel_unittest'], stdin=parent_schema_handle) + call(['mysql', '-u', 'root', 'apel_unittest'], stdin=parent_schema_handle) parent_schema_handle.close() schema_path = os.path.abspath(os.path.join('..', 'schemas', schema)) schema_handle = open(schema_path) - p = Popen(['mysql', 'apel_unittest'], stdin=schema_handle, stderr=PIPE) + p = Popen(['mysql', '-u', 'root', 'apel_unittest'], stdin=schema_handle, stderr=PIPE) schema_handle.close() p.wait() self.assertFalse(p.returncode, p.communicate()[1]) From bd63d73dd4c8b84897756d57c8b457fb3b3f883e Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 17 Jun 2015 11:09:04 +0100 Subject: [PATCH 27/45] Tidy up query part of mysql call --- test/test_mysql.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_mysql.py b/test/test_mysql.py index 54ac7cc78..59bd43459 100644 --- a/test/test_mysql.py +++ b/test/test_mysql.py @@ -28,7 +28,8 @@ def setUp(self): 'apel_unittest') def tearDown(self): - subprocess.call(['mysql', '-u', 'root', '-e', 'DROP DATABASE apel_unittest;']) + query = "DROP DATABASE apel_unittest;" + subprocess.call(['mysql', '-u', 'root', '-e', query]) def test_test_connection(self): """Basic check that test_connection works without error.""" From 2cfed7e05604681aae47bfec9639b3da3e6b3c80 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 17 Jun 2015 11:11:03 +0100 Subject: [PATCH 28/45] Disable code that runs slowly on Travis CI --- test/test_mysql.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/test_mysql.py b/test/test_mysql.py index 59bd43459..15cfa42ef 100644 --- a/test/test_mysql.py +++ b/test/test_mysql.py @@ -27,9 +27,10 @@ def setUp(self): self.db = apel.db.apeldb.ApelDb('mysql', 'localhost', 3306, 'root', '', 'apel_unittest') - def tearDown(self): - query = "DROP DATABASE apel_unittest;" - subprocess.call(['mysql', '-u', 'root', '-e', query]) + # This method seems to run really slowly on Travis CI + #def tearDown(self): + # query = "DROP DATABASE apel_unittest;" + # subprocess.call(['mysql', '-u', 'root', '-e', query]) def test_test_connection(self): """Basic check that test_connection works without error.""" From 78f0060c4757f8f832d31746d8b752e118fbdf0d Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 17 Jun 2015 11:30:37 +0100 Subject: [PATCH 29/45] Add missing buffer option to unit test run --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ccbdff081..df242a1f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,6 @@ before_script: - cd test # Command to run tests -script: coverage run --source=apel,bin -m unittest2 discover +script: coverage run --source=apel,bin -m unittest2 discover --buffer after_success: coveralls From 173072b6f461444b95b5bf73dd2beb5bfe34dc3f Mon Sep 17 00:00:00 2001 From: Pavel Demin Date: Wed, 19 Nov 2014 13:16:12 +0000 Subject: [PATCH 30/45] Initial commit of HTCondor parser files --- apel/parsers/htcondor.py | 70 ++++++++++++++++++++++++++++++++++++++++ scripts/htcondor_acc.sh | 13 ++++++++ 2 files changed, 83 insertions(+) create mode 100644 apel/parsers/htcondor.py create mode 100644 scripts/htcondor_acc.sh diff --git a/apel/parsers/htcondor.py b/apel/parsers/htcondor.py new file mode 100644 index 000000000..204756785 --- /dev/null +++ b/apel/parsers/htcondor.py @@ -0,0 +1,70 @@ +''' + Copyright (C) 2014 STFC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + @author: Pavel Demin +''' +from apel.common import parse_time +from apel.db.records.event import EventRecord +from apel.parsers import Parser + +import time +import datetime +import logging + +log = logging.getLogger(__name__) + + +class HTCondorParser(Parser): + ''' + First implementation of the APEL parser for HTCondor + ''' + def __init__(self, site, machine_name, mpi): + Parser.__init__(self, site, machine_name, mpi) + log.info('Site: %s; batch system: %s' % (self.site_name, self.machine_name)) + + def parse(self, line): + ''' + Parses single line from accounting log file. + ''' + # condor_history -constraint "JobStartDate > 0" -format "%d|" ClusterId -format "%s|" Owner -format "%d|" RemoteWallClockTime -format "%d|" RemoteUserCpu -format "%d|" RemoteSysCpu -format "%d|" JobStartDate -format "%d|" EnteredCurrentStatus -format "%d|" ResidentSetSize_RAW -format "%d|" ImageSize_RAW -format "%d\n" RequestCpus + # 1234567|opssgm|19|18|8|1412688273|1412708199|712944|2075028|1 + + values = line.strip().split('|') + + mapping = {'Site' : lambda x: self.site_name, + 'MachineName' : lambda x: self.machine_name, + 'Infrastructure' : lambda x: "APEL-CREAM-HTCONDOR", + 'JobName' : lambda x: x[0], + 'LocalUserID' : lambda x: x[1], + 'LocalUserGroup' : lambda x: "", + 'WallDuration' : lambda x: x[2], + 'CpuDuration' : lambda x: x[3]+x[4], + 'StartTime' : lambda x: x[5], + 'StopTime' : lambda x: x[6], + 'MemoryReal' : lambda x: x[7], + 'MemoryVirtual' : lambda x: x[8], + 'Processors' : lambda x: x[9], + 'NodeCount' : lambda x: 0 + } + + rc = {} + + for key in mapping: + rc[key] = mapping[key](values) + + record = EventRecord() + record.set_all(rc) + return record + diff --git a/scripts/htcondor_acc.sh b/scripts/htcondor_acc.sh new file mode 100644 index 000000000..5c8a9e17e --- /dev/null +++ b/scripts/htcondor_acc.sh @@ -0,0 +1,13 @@ +#! /bin/sh + +CONDOR_LOCATION=/home/condor/release +OUTPUT_LOCATION=/scratch/condor/apel + +NOW=$(date +"%Y%m%dT%H%M%S") +OUTPUT_FILE=$OUTPUT_LOCATION/accounting.$NOW + +$CONDOR_LOCATION/bin/condor_history -constraint "JobStartDate > 0" -format "%d|" ClusterId -format "%s|" Owner -format "%d|" RemoteWallClockTime -format "%d|" RemoteUserCpu -format "%d|" RemoteSysCpu -format "%d|" JobStartDate -format "%d|" EnteredCurrentStatus -format "%d|" ResidentSetSize_RAW -format "%d|" ImageSize_RAW -format "%d\n" RequestCpus > $OUTPUT_FILE + +/usr/bin/apelparser + +/bin/find $OUTPUT_LOCATION -name accounting.\* -mtime +30 -exec /bin/rm {} \; From 36abc38c1d04892dfe7d0635fd1c299d9c858463 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 19 Nov 2014 14:15:48 +0000 Subject: [PATCH 31/45] Add unit test for HTCondor parser - Add unit test for HTCondor parser which just checks the basic parser as initially committed. - Tidy up the parser a little. --- apel/parsers/htcondor.py | 11 +++---- test/test_htcondor_parser.py | 59 ++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 test/test_htcondor_parser.py diff --git a/apel/parsers/htcondor.py b/apel/parsers/htcondor.py index 204756785..5e9c82fae 100644 --- a/apel/parsers/htcondor.py +++ b/apel/parsers/htcondor.py @@ -1,5 +1,5 @@ ''' - Copyright (C) 2014 STFC + Copyright 2014 The Science and Technology Facilities Council Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -15,13 +15,13 @@ @author: Pavel Demin ''' -from apel.common import parse_time + + +import logging + from apel.db.records.event import EventRecord from apel.parsers import Parser -import time -import datetime -import logging log = logging.getLogger(__name__) @@ -67,4 +67,3 @@ def parse(self, line): record = EventRecord() record.set_all(rc) return record - diff --git a/test/test_htcondor_parser.py b/test/test_htcondor_parser.py new file mode 100644 index 000000000..66c9157fc --- /dev/null +++ b/test/test_htcondor_parser.py @@ -0,0 +1,59 @@ +import datetime +import unittest + +from apel.parsers.htcondor import HTCondorParser + + +class HTCondorParserTest(unittest.TestCase): + """ + Test case for HTCondor parser + """ + + def setUp(self): + self.parser = HTCondorParser('testSite', 'testHost', True) + + def test_parse_line(self): + + keys = ('JobName', 'LocalUserID', 'LocalUserGroup', 'WallDuration', + 'CpuDuration', 'StartTime', 'StopTime', 'MemoryReal', + 'MemoryVirtual', 'NodeCount', 'Processors') + + # condor_history output (line split): + # ClusterId|Owner|RemoteWallClockTime|RemoteUserCpu|RemoteSysCpu| + # JobStartDate|EnteredCurrentStatus|ResidentSetSize_RAW|ImageSize_RAW| + # RequestCpus + + # Examples for correct lines + lines = ( + ('1234567|opssgm|19|18|8|1412688273|1412708199|712944|2075028|1'), + ) + + values = ( + ('1234567', 'opssgm', None, 19, 188, + datetime.datetime(2014, 10, 7, 13, 24, 33), + datetime.datetime(2014, 10, 7, 18, 56, 39), + 712944, 2075028, 0, 1), + ) + + cases = {} + for line, value in zip(lines, values): + cases[line] = dict(zip(keys, value)) + + for line in cases.keys(): + record = self.parser.parse(line) + cont = record._record_content + + self.assertEqual(cont['Site'], 'testSite') + self.assertEqual(cont['MachineName'], 'testHost') + self.assertEqual(cont['Infrastructure'], 'APEL-CREAM-HTCONDOR') + + for key in cases[line].keys(): + self.assertTrue(key in cont, "Key '%s' not in record." % key) + + for key in cases[line].keys(): + self.assertEqual(cont[key], cases[line][key], + "%s != %s for key %s." % + (cont[key], cases[line][key], key)) + +if __name__ == '__main__': + unittest.main() From c998da3d10db5031f7589df692fc4db342f2fe3a Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 19 Nov 2014 14:41:35 +0000 Subject: [PATCH 32/45] Cast integers to ints when parsing - Cast integers to ints when parsing to avoid string related problems. - Correct unit test to match. --- apel/parsers/htcondor.py | 10 +++++----- test/test_htcondor_parser.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apel/parsers/htcondor.py b/apel/parsers/htcondor.py index 5e9c82fae..b7232ccc0 100644 --- a/apel/parsers/htcondor.py +++ b/apel/parsers/htcondor.py @@ -49,13 +49,13 @@ def parse(self, line): 'JobName' : lambda x: x[0], 'LocalUserID' : lambda x: x[1], 'LocalUserGroup' : lambda x: "", - 'WallDuration' : lambda x: x[2], - 'CpuDuration' : lambda x: x[3]+x[4], + 'WallDuration' : lambda x: int(x[2]), + 'CpuDuration' : lambda x: int(x[3])+int(x[4]), 'StartTime' : lambda x: x[5], 'StopTime' : lambda x: x[6], - 'MemoryReal' : lambda x: x[7], - 'MemoryVirtual' : lambda x: x[8], - 'Processors' : lambda x: x[9], + 'MemoryReal' : lambda x: int(x[7]), + 'MemoryVirtual' : lambda x: int(x[8]), + 'Processors' : lambda x: int(x[9]), 'NodeCount' : lambda x: 0 } diff --git a/test/test_htcondor_parser.py b/test/test_htcondor_parser.py index 66c9157fc..8a2300fbd 100644 --- a/test/test_htcondor_parser.py +++ b/test/test_htcondor_parser.py @@ -29,7 +29,7 @@ def test_parse_line(self): ) values = ( - ('1234567', 'opssgm', None, 19, 188, + ('1234567', 'opssgm', None, 19, 26, datetime.datetime(2014, 10, 7, 13, 24, 33), datetime.datetime(2014, 10, 7, 18, 56, 39), 712944, 2075028, 0, 1), From 84c29c5268440700b16619e7d542e1904661409a Mon Sep 17 00:00:00 2001 From: David Rebatto Date: Tue, 24 Mar 2015 11:32:56 +0000 Subject: [PATCH 33/45] Add group to condor parser and expand bash script --- apel/parsers/htcondor.py | 2 +- scripts/htcondor_acc.sh | 36 +++++++++++++++++++++++++++++++++--- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/apel/parsers/htcondor.py b/apel/parsers/htcondor.py index b7232ccc0..954b7218c 100644 --- a/apel/parsers/htcondor.py +++ b/apel/parsers/htcondor.py @@ -48,7 +48,7 @@ def parse(self, line): 'Infrastructure' : lambda x: "APEL-CREAM-HTCONDOR", 'JobName' : lambda x: x[0], 'LocalUserID' : lambda x: x[1], - 'LocalUserGroup' : lambda x: "", + 'LocalUserGroup' : lambda x: x[10], 'WallDuration' : lambda x: int(x[2]), 'CpuDuration' : lambda x: int(x[3])+int(x[4]), 'StartTime' : lambda x: x[5], diff --git a/scripts/htcondor_acc.sh b/scripts/htcondor_acc.sh index 5c8a9e17e..d40f1cc6a 100644 --- a/scripts/htcondor_acc.sh +++ b/scripts/htcondor_acc.sh @@ -1,13 +1,43 @@ #! /bin/sh -CONDOR_LOCATION=/home/condor/release -OUTPUT_LOCATION=/scratch/condor/apel +CONDOR_LOCATION=/usr +OUTPUT_LOCATION=/var/log/accounting +# Find all the history files modified in the last two months +# (there can be more than one, if the CE submits to several schedds) +HISTORY_FILES=$(find /var/lib/condor/spool/ -name history\* -mtime -61) + +# Create a temporary accounting file name NOW=$(date +"%Y%m%dT%H%M%S") OUTPUT_FILE=$OUTPUT_LOCATION/accounting.$NOW -$CONDOR_LOCATION/bin/condor_history -constraint "JobStartDate > 0" -format "%d|" ClusterId -format "%s|" Owner -format "%d|" RemoteWallClockTime -format "%d|" RemoteUserCpu -format "%d|" RemoteSysCpu -format "%d|" JobStartDate -format "%d|" EnteredCurrentStatus -format "%d|" ResidentSetSize_RAW -format "%d|" ImageSize_RAW -format "%d\n" RequestCpus > $OUTPUT_FILE +# Build the filter for the history command +CONSTR="(JobStartDate>0)&&(CompletionDate>`date +%s -d "01 Jan 2014"`)" + +# Only get the jobs submitted from this CE (ceCertificateSubject is +# set in /etc/condor/config.s/blah and added to the job classad with +# the SUBMIT_EXPRS directive in the same file) +CONSTR=${CONSTR}"&&(ceCertificateSubject==$(condor_config_val ceCertificateSubject))" + +# Populate the temporary file +for HF in $HISTORY_FILES +do + $CONDOR_LOCATION/bin/condor_history -file $HF -constraint $CONSTR \ + -format "%s|" GlobalJobId \ + -format "%s|" Owner \ + -format "%d|" RemoteWallClockTime \ + -format "%d|" RemoteUserCpu \ + -format "%d|" RemoteSysCpu \ + -format "%d|" JobStartDate \ + -format "%d|" EnteredCurrentStatus \ + -format "%d|" ResidentSetSize_RAW \ + -format "%d|" ImageSize_RAW \ + -format "%d|" RequestCpus \ + -format "%s\n" Group >> $OUTPUT_FILE +done +# Invoke the parser /usr/bin/apelparser +# Cleanup /bin/find $OUTPUT_LOCATION -name accounting.\* -mtime +30 -exec /bin/rm {} \; From 3bb9af75f4ae09c041dcaf37379a8738891be5e0 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 23 Jun 2015 11:34:59 +0100 Subject: [PATCH 34/45] Remove section that refers to site-specific config --- scripts/htcondor_acc.sh | 5 ----- test/test_htcondor_parser.py | 1 + 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/htcondor_acc.sh b/scripts/htcondor_acc.sh index d40f1cc6a..0c89c7016 100644 --- a/scripts/htcondor_acc.sh +++ b/scripts/htcondor_acc.sh @@ -14,11 +14,6 @@ OUTPUT_FILE=$OUTPUT_LOCATION/accounting.$NOW # Build the filter for the history command CONSTR="(JobStartDate>0)&&(CompletionDate>`date +%s -d "01 Jan 2014"`)" -# Only get the jobs submitted from this CE (ceCertificateSubject is -# set in /etc/condor/config.s/blah and added to the job classad with -# the SUBMIT_EXPRS directive in the same file) -CONSTR=${CONSTR}"&&(ceCertificateSubject==$(condor_config_val ceCertificateSubject))" - # Populate the temporary file for HF in $HISTORY_FILES do diff --git a/test/test_htcondor_parser.py b/test/test_htcondor_parser.py index 8a2300fbd..d9d8b0027 100644 --- a/test/test_htcondor_parser.py +++ b/test/test_htcondor_parser.py @@ -55,5 +55,6 @@ def test_parse_line(self): "%s != %s for key %s." % (cont[key], cases[line][key], key)) + if __name__ == '__main__': unittest.main() From 41e1ed334fea212878af773b3a408fc3d6756ba4 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 6 Jul 2015 15:17:58 +0100 Subject: [PATCH 35/45] Remove addition of non-standard group extraction --- apel/parsers/htcondor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apel/parsers/htcondor.py b/apel/parsers/htcondor.py index 954b7218c..b7232ccc0 100644 --- a/apel/parsers/htcondor.py +++ b/apel/parsers/htcondor.py @@ -48,7 +48,7 @@ def parse(self, line): 'Infrastructure' : lambda x: "APEL-CREAM-HTCONDOR", 'JobName' : lambda x: x[0], 'LocalUserID' : lambda x: x[1], - 'LocalUserGroup' : lambda x: x[10], + 'LocalUserGroup' : lambda x: "", 'WallDuration' : lambda x: int(x[2]), 'CpuDuration' : lambda x: int(x[3])+int(x[4]), 'StartTime' : lambda x: x[5], From d5c61fe36f6ea9da70197f048c51d2eded1a8dbd Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 6 Jul 2015 15:19:08 +0100 Subject: [PATCH 36/45] Add more tests for htcondor parser --- test/test_htcondor_parser.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/test/test_htcondor_parser.py b/test/test_htcondor_parser.py index d9d8b0027..6928bddf6 100644 --- a/test/test_htcondor_parser.py +++ b/test/test_htcondor_parser.py @@ -25,7 +25,10 @@ def test_parse_line(self): # Examples for correct lines lines = ( - ('1234567|opssgm|19|18|8|1412688273|1412708199|712944|2075028|1'), + '1234567|opssgm|19|18|8|1412688273|1412708199|712944|2075028|1', + 'arcce.rl.ac.uk#2376.0#1589|tatls011|287|107|11|1435671643|1435671930|26636|26832|1|1', + 'arcce.rl.ac.uk#2486.0#1888|t2k016|4|0|0|1435671919|1435671922|0|11|1|1', + 'arcce.rl.ac.uk#2478.0#1874|snoplus014|2|0|0|1435671918|1435671920|0|11|1|1', ) values = ( @@ -33,6 +36,18 @@ def test_parse_line(self): datetime.datetime(2014, 10, 7, 13, 24, 33), datetime.datetime(2014, 10, 7, 18, 56, 39), 712944, 2075028, 0, 1), + ('arcce.rl.ac.uk#2376.0#1589', 'tatls011', None, 287, 118, + datetime.datetime(2015, 6, 30, 13, 40, 43), + datetime.datetime(2015, 6, 30, 13, 45, 30), + 26636, 26832, 0, 1), + ('arcce.rl.ac.uk#2486.0#1888', 't2k016', None, 4, 0, + datetime.datetime(2015, 6, 30, 13, 45, 19), + datetime.datetime(2015, 6, 30, 13, 45, 22), + 0, 11, 0, 1), + ('arcce.rl.ac.uk#2478.0#1874', 'snoplus014', None, 2, 0, + datetime.datetime(2015, 6, 30, 13, 45, 18), + datetime.datetime(2015, 6, 30, 13, 45, 20), + 0, 11, 0, 1), ) cases = {} From 70cd745030d31ebaa1a8ee483b59ba41811b2cae Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 6 Jul 2015 15:19:39 +0100 Subject: [PATCH 37/45] Remove group from htcondor script - Remove Group from htcondor script as it's non-standard. --- scripts/htcondor_acc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/htcondor_acc.sh b/scripts/htcondor_acc.sh index 0c89c7016..0dcf43823 100644 --- a/scripts/htcondor_acc.sh +++ b/scripts/htcondor_acc.sh @@ -28,7 +28,7 @@ do -format "%d|" ResidentSetSize_RAW \ -format "%d|" ImageSize_RAW \ -format "%d|" RequestCpus \ - -format "%s\n" Group >> $OUTPUT_FILE + -format "\n" EMPTY >> $OUTPUT_FILE done # Invoke the parser From 2288c0c3a469f77cc5178376cc97399447abb50c Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 6 Jul 2015 15:24:08 +0100 Subject: [PATCH 38/45] Update comments to match condor command --- apel/parsers/htcondor.py | 4 ++-- test/test_htcondor_parser.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apel/parsers/htcondor.py b/apel/parsers/htcondor.py index b7232ccc0..1e3e8d212 100644 --- a/apel/parsers/htcondor.py +++ b/apel/parsers/htcondor.py @@ -38,8 +38,8 @@ def parse(self, line): ''' Parses single line from accounting log file. ''' - # condor_history -constraint "JobStartDate > 0" -format "%d|" ClusterId -format "%s|" Owner -format "%d|" RemoteWallClockTime -format "%d|" RemoteUserCpu -format "%d|" RemoteSysCpu -format "%d|" JobStartDate -format "%d|" EnteredCurrentStatus -format "%d|" ResidentSetSize_RAW -format "%d|" ImageSize_RAW -format "%d\n" RequestCpus - # 1234567|opssgm|19|18|8|1412688273|1412708199|712944|2075028|1 + # condor_history -constraint "JobStartDate > 0" -format "%s|" GlobalJobId -format "%s|" Owner -format "%d|" RemoteWallClockTime -format "%d|" RemoteUserCpu -format "%d|" RemoteSysCpu -format "%d|" JobStartDate -format "%d|" EnteredCurrentStatus -format "%d|" ResidentSetSize_RAW -format "%d|" ImageSize_RAW -format "%d|" RequestCpus -format "%s\n" RequestCpus + # arcce.rl.ac.uk#2376.0#71589|tatls011|287|107|11|1435671643|1435671930|26636|26832|1|1 values = line.strip().split('|') diff --git a/test/test_htcondor_parser.py b/test/test_htcondor_parser.py index 6928bddf6..cd3738aa7 100644 --- a/test/test_htcondor_parser.py +++ b/test/test_htcondor_parser.py @@ -19,7 +19,7 @@ def test_parse_line(self): 'MemoryVirtual', 'NodeCount', 'Processors') # condor_history output (line split): - # ClusterId|Owner|RemoteWallClockTime|RemoteUserCpu|RemoteSysCpu| + # GlobalJobId|Owner|RemoteWallClockTime|RemoteUserCpu|RemoteSysCpu| # JobStartDate|EnteredCurrentStatus|ResidentSetSize_RAW|ImageSize_RAW| # RequestCpus From 08762ae2a36f5f64d75caf5189cbe29b672458cb Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 6 Jul 2015 15:46:17 +0100 Subject: [PATCH 39/45] Add HTCondor as an option to parser executable --- bin/parser.py | 5 ++++- conf/parser.cfg | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/parser.py b/bin/parser.py index 08c3e98b5..2c328ef3d 100644 --- a/bin/parser.py +++ b/bin/parser.py @@ -43,6 +43,7 @@ from apel.parsers.sge import SGEParser from apel.parsers.pbs import PBSParser from apel.parsers.slurm import SlurmParser +from apel.parsers.htcondor import HTCondorParser LOGGER_ID = 'parser' @@ -55,9 +56,11 @@ 'LSF': LSFParser, 'SGE': SGEParser, 'SLURM': SlurmParser, - 'blah' : BlahParser + 'blah' : BlahParser, + 'HTCondor': HTCondorParser, } + class ParserConfigException(Exception): ''' Exception raised when parser is misconfigured. diff --git a/conf/parser.cfg b/conf/parser.cfg index 41329d8d6..a5643aec9 100644 --- a/conf/parser.cfg +++ b/conf/parser.cfg @@ -29,7 +29,7 @@ enabled = true reparse = false # Batch system specific options. -# Valid types are LSF, PBS, SGE, SLURM +# Valid types are LSF, PBS, SGE, SLURM, HTCondor type = # Whether to try to parse multi-core details parallel = true From c8785b09934cf8f9a97de3d1c77aedd52d928953 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 14 Jul 2015 13:50:41 +0100 Subject: [PATCH 40/45] Increase period to parse Condor history files from - Increase the period to select Condor history files from for parsing from 61 to 62 days to cover having two adjacent 31 day months. i.e. July + August, December + January. --- scripts/htcondor_acc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/htcondor_acc.sh b/scripts/htcondor_acc.sh index 0dcf43823..fe7977175 100644 --- a/scripts/htcondor_acc.sh +++ b/scripts/htcondor_acc.sh @@ -5,7 +5,7 @@ OUTPUT_LOCATION=/var/log/accounting # Find all the history files modified in the last two months # (there can be more than one, if the CE submits to several schedds) -HISTORY_FILES=$(find /var/lib/condor/spool/ -name history\* -mtime -61) +HISTORY_FILES=$(find /var/lib/condor/spool/ -name history\* -mtime -62) # Create a temporary accounting file name NOW=$(date +"%Y%m%dT%H%M%S") From d413180d8d26b47b9290b5825ef8e7d8794f8322 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 14 Jul 2015 14:01:04 +0100 Subject: [PATCH 41/45] Add explicit config arg to Condor apelparser call --- scripts/htcondor_acc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/htcondor_acc.sh b/scripts/htcondor_acc.sh index fe7977175..c31f0398d 100644 --- a/scripts/htcondor_acc.sh +++ b/scripts/htcondor_acc.sh @@ -32,7 +32,7 @@ do done # Invoke the parser -/usr/bin/apelparser +/usr/bin/apelparser --config /etc/apel/parser.cfg # Cleanup /bin/find $OUTPUT_LOCATION -name accounting.\* -mtime +30 -exec /bin/rm {} \; From 6b83c42795a9aca150ef834c642783d83c675b3e Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 14 Jul 2015 14:05:42 +0100 Subject: [PATCH 42/45] Make command in htcondor.py comment match script --- apel/parsers/htcondor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apel/parsers/htcondor.py b/apel/parsers/htcondor.py index 1e3e8d212..03e03e409 100644 --- a/apel/parsers/htcondor.py +++ b/apel/parsers/htcondor.py @@ -38,7 +38,7 @@ def parse(self, line): ''' Parses single line from accounting log file. ''' - # condor_history -constraint "JobStartDate > 0" -format "%s|" GlobalJobId -format "%s|" Owner -format "%d|" RemoteWallClockTime -format "%d|" RemoteUserCpu -format "%d|" RemoteSysCpu -format "%d|" JobStartDate -format "%d|" EnteredCurrentStatus -format "%d|" ResidentSetSize_RAW -format "%d|" ImageSize_RAW -format "%d|" RequestCpus -format "%s\n" RequestCpus + # condor_history -constraint "JobStartDate > 0" -format "%s|" GlobalJobId -format "%s|" Owner -format "%d|" RemoteWallClockTime -format "%d|" RemoteUserCpu -format "%d|" RemoteSysCpu -format "%d|" JobStartDate -format "%d|" EnteredCurrentStatus -format "%d|" ResidentSetSize_RAW -format "%d|" ImageSize_RAW -format "%d|" RequestCpus # arcce.rl.ac.uk#2376.0#71589|tatls011|287|107|11|1435671643|1435671930|26636|26832|1|1 values = line.strip().split('|') From 90ec0766e58fa4347cd6987e03799eee01c70bea Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 14 Jul 2015 14:57:53 +0100 Subject: [PATCH 43/45] Add htcondor_acc script to spec file --- apel.spec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apel.spec b/apel.spec index 225d3199a..5bef24823 100644 --- a/apel.spec +++ b/apel.spec @@ -109,8 +109,9 @@ cp schemas/server-extra.sql %{buildroot}%_datadir/apel/ cp schemas/cloud.sql %{buildroot}%_datadir/apel/ cp schemas/storage.sql %{buildroot}%_datadir/apel/ -# slurm accounting script +# accounting scripts cp scripts/slurm_acc.sh %{buildroot}%_datadir/apel/ +cp scripts/htcondor_acc.sh %{buildroot}%_datadir/apel/ # message status script cp scripts/msg_status.py %{buildroot}%_datadir/apel/ From 85c494f6577d01e95c15e52c8a8572d501a890ee Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 14 Jul 2015 15:04:18 +0100 Subject: [PATCH 44/45] Update changelogs and version nums for 1.5.0 --- CHANGELOG | 7 +++++++ apel.spec | 15 +++++++++++---- apel/__init__.py | 2 +- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index c4e80fcbe..0a142d2ca 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,12 @@ Changelog for apel ================== + * Tue Jul 14 2015 Adrian Coveney - 1.5.0-1 + - Added sorting of accounting logs before parsing which makes reading through + the parser log easier, especially if files use the YYYYMMDD date format. + - Added the first version of a basic HTCondor parser. + - Fixed the server schema to correctly aggregate super summaries when + viewed as normalised super summaries. + * Thu Mar 12 2015 Adrian Coveney - 1.4.1-1 - Changed defaults so that parallel jobs are reported for new installations. - Corrected year calculation in migrate_apel.py script. diff --git a/apel.spec b/apel.spec index 5bef24823..2bd0ead93 100644 --- a/apel.spec +++ b/apel.spec @@ -4,7 +4,7 @@ %endif Name: apel -Version: 1.4.1 +Version: 1.5.0 %define releasenumber 1 Release: %{releasenumber}%{?dist} Summary: APEL packages @@ -34,7 +34,7 @@ apel-lib provides required libraries for the rest of APEL system. %package parsers Summary: Parsers for APEL system Group: Development/Languages -Requires: apel-lib >= 1.3.1 +Requires: apel-lib >= 1.5.0 Requires(pre): shadow-utils %description parsers @@ -44,7 +44,7 @@ supported by the APEL system: Torque, SGE and LSF. %package client Summary: APEL client package Group: Development/Languages -Requires: apel-lib >= 1.3.1, apel-ssm +Requires: apel-lib >= 1.5.0, apel-ssm Requires(pre): shadow-utils %description client @@ -55,7 +55,7 @@ SSM. %package server Summary: APEL server package Group: Development/Languages -Requires: apel-lib >= 1.3.1, apel-ssm +Requires: apel-lib >= 1.5.0, apel-ssm Requires(pre): shadow-utils %description server @@ -188,6 +188,13 @@ exit 0 # ============================================================================== %changelog + * Tue Jul 14 2015 Adrian Coveney - 1.5.0-1 + - Added sorting of accounting logs before parsing which makes reading through + the parser log easier, especially if files use the YYYYMMDD date format. + - Added the first version of a basic HTCondor parser. + - Fixed the server schema to correctly aggregate super summaries when + viewed as normalised super summaries. + * Thu Mar 12 2015 Adrian Coveney - 1.4.1-1 - Changed defaults so that parallel jobs are reported for new installations. - Corrected year calculation in migrate_apel.py script. diff --git a/apel/__init__.py b/apel/__init__.py index 7c6e6ba65..1e96bd9dd 100644 --- a/apel/__init__.py +++ b/apel/__init__.py @@ -15,4 +15,4 @@ @author Konrad Jopek, Will Rogers ''' -__version__ = (1, 4, 1) +__version__ = (1, 5, 0) From 6611c8b2127bbf4ce694b466b1fa56d95cffbaa0 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 14 Jul 2015 15:21:22 +0100 Subject: [PATCH 45/45] Add htcondor_acc script to parser rpm --- apel.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/apel.spec b/apel.spec index 2bd0ead93..516a96cab 100644 --- a/apel.spec +++ b/apel.spec @@ -141,6 +141,7 @@ exit 0 %attr(755,root,root) %_bindir/apelparser %config(noreplace) %attr(600,-,-) %{apelconf}/parser.cfg %attr(755,root,root) %_datadir/apel/slurm_acc.sh +%attr(755,root,root) %_datadir/apel/htcondor_acc.sh # ------------------------------------------------------------------------------