Merge branch 'release-1.5.0'

.travis.yml

@@ -9,20 +9,19 @@ matrix:
     - python: "3.4"
   fast_finish: true
 
-# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
-install:
-  - pip install MySQL-python
-  - pip install python-ldap
-  - pip install iso8601
-  - pip install dirq
-  - pip install unittest2
-  - pip install coveralls
+# Route build to container-based infrastructure
+sudo: false
+# Cache the dependencies installed by pip
+cache: pip
 
-# command to run tests, e.g. python setup.py test
-script:
+# Install defaults to "pip install -r requirements.txt"
+
+# Commands that prepare things for the test
+before_script:
   - export PYTHONPATH=$PYTHONPATH:`pwd -P`
   - cd test
-  - coverage run --source=apel,bin -m unittest2 discover
 
-after_success:
-  - coveralls
+# Command to run tests
+script: coverage run --source=apel,bin -m unittest2 discover --buffer
+
+after_success: coveralls

CHANGELOG

@@ -1,5 +1,12 @@
 Changelog for apel
 ==================
+ * Tue Jul 14 2015 Adrian Coveney <adrian.coveney@stfc.ac.uk> - 1.5.0-1
+ - Added sorting of accounting logs before parsing which makes reading through
+   the parser log easier, especially if files use the YYYYMMDD date format.
+ - Added the first version of a basic HTCondor parser.
+ - Fixed the server schema to correctly aggregate super summaries when
+   viewed as normalised super summaries.
+
  * Thu Mar 12 2015 Adrian Coveney <adrian.coveney@stfc.ac.uk> - 1.4.1-1
  - Changed defaults so that parallel jobs are reported for new installations.
  - Corrected year calculation in migrate_apel.py script.

apel.spec

@@ -4,7 +4,7 @@
 %endif
 
 Name:           apel
-Version:        1.4.1
+Version:        1.5.0
 %define releasenumber 1
 Release:        %{releasenumber}%{?dist}
 Summary:        APEL packages
@@ -34,7 +34,7 @@ apel-lib provides required libraries for the rest of APEL system.
 %package parsers
 Summary:        Parsers for APEL system
 Group:          Development/Languages
-Requires:       apel-lib >= 1.3.1
+Requires:       apel-lib >= 1.5.0
 Requires(pre):  shadow-utils
 
 %description parsers
@@ -44,7 +44,7 @@ supported by the APEL system: Torque, SGE and LSF.
 %package client
 Summary:        APEL client package
 Group:          Development/Languages
-Requires:       apel-lib >= 1.3.1, apel-ssm
+Requires:       apel-lib >= 1.5.0, apel-ssm
 Requires(pre):  shadow-utils
 
 %description client
@@ -55,7 +55,7 @@ SSM.
 %package server
 Summary:        APEL server package
 Group:          Development/Languages
-Requires:       apel-lib >= 1.3.1, apel-ssm
+Requires:       apel-lib >= 1.5.0, apel-ssm
 Requires(pre):  shadow-utils
 
 %description server
@@ -109,8 +109,9 @@ cp schemas/server-extra.sql %{buildroot}%_datadir/apel/
 cp schemas/cloud.sql %{buildroot}%_datadir/apel/
 cp schemas/storage.sql %{buildroot}%_datadir/apel/
 
-# slurm accounting script
+# accounting scripts
 cp scripts/slurm_acc.sh %{buildroot}%_datadir/apel/
+cp scripts/htcondor_acc.sh %{buildroot}%_datadir/apel/
 
 # message status script
 cp scripts/msg_status.py %{buildroot}%_datadir/apel/
@@ -140,6 +141,7 @@ exit 0
 %attr(755,root,root) %_bindir/apelparser
 %config(noreplace) %attr(600,-,-) %{apelconf}/parser.cfg
 %attr(755,root,root) %_datadir/apel/slurm_acc.sh
+%attr(755,root,root) %_datadir/apel/htcondor_acc.sh
 
 # ------------------------------------------------------------------------------
 
@@ -187,6 +189,13 @@ exit 0
 # ==============================================================================
 
 %changelog
+ * Tue Jul 14 2015 Adrian Coveney <adrian.coveney@stfc.ac.uk> - 1.5.0-1
+ - Added sorting of accounting logs before parsing which makes reading through
+   the parser log easier, especially if files use the YYYYMMDD date format.
+ - Added the first version of a basic HTCondor parser.
+ - Fixed the server schema to correctly aggregate super summaries when
+   viewed as normalised super summaries.
+
  * Thu Mar 12 2015 Adrian Coveney <adrian.coveney@stfc.ac.uk> - 1.4.1-1
  - Changed defaults so that parallel jobs are reported for new installations.
  - Corrected year calculation in migrate_apel.py script.

apel/__init__.py

@@ -15,4 +15,4 @@
    
    @author Konrad Jopek, Will Rogers
 '''
-__version__ = (1, 4, 1)
+__version__ = (1, 5, 0)

apel/parsers/htcondor.py

@@ -0,0 +1,69 @@
+'''
+   Copyright 2014 The Science and Technology Facilities Council
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+   @author: Pavel Demin
+'''
+
+
+import logging
+
+from apel.db.records.event import EventRecord
+from apel.parsers import Parser
+
+
+log = logging.getLogger(__name__)
+
+
+class HTCondorParser(Parser):
+    '''
+    First implementation of the APEL parser for HTCondor
+    '''
+    def __init__(self, site, machine_name, mpi):
+        Parser.__init__(self, site, machine_name, mpi)
+        log.info('Site: %s; batch system: %s' % (self.site_name, self.machine_name))
+
+    def parse(self, line):
+        '''
+        Parses single line from accounting log file.
+        '''
+        # condor_history -constraint "JobStartDate > 0" -format "%s|" GlobalJobId -format "%s|" Owner -format "%d|" RemoteWallClockTime -format "%d|" RemoteUserCpu -format "%d|" RemoteSysCpu -format "%d|" JobStartDate -format "%d|" EnteredCurrentStatus -format "%d|" ResidentSetSize_RAW -format "%d|" ImageSize_RAW -format "%d|" RequestCpus
+        # arcce.rl.ac.uk#2376.0#71589|tatls011|287|107|11|1435671643|1435671930|26636|26832|1|1
+
+        values = line.strip().split('|')
+
+        mapping = {'Site'            : lambda x: self.site_name,
+                   'MachineName'     : lambda x: self.machine_name,
+                   'Infrastructure'  : lambda x: "APEL-CREAM-HTCONDOR",
+                   'JobName'         : lambda x: x[0],
+                   'LocalUserID'     : lambda x: x[1],
+                   'LocalUserGroup'  : lambda x: "",
+                   'WallDuration'    : lambda x: int(x[2]),
+                   'CpuDuration'     : lambda x: int(x[3])+int(x[4]),
+                   'StartTime'       : lambda x: x[5],
+                   'StopTime'        : lambda x: x[6],
+                   'MemoryReal'      : lambda x: int(x[7]),
+                   'MemoryVirtual'   : lambda x: int(x[8]),
+                   'Processors'      : lambda x: int(x[9]),
+                   'NodeCount'       : lambda x: 0
+                  }
+
+        rc = {}
+
+        for key in mapping:
+            rc[key] = mapping[key](values)
+
+        record = EventRecord()
+        record.set_all(rc)
+        return record

bin/parser.py

@@ -43,6 +43,7 @@
 from apel.parsers.sge import SGEParser
 from apel.parsers.pbs import PBSParser
 from apel.parsers.slurm import SlurmParser
+from apel.parsers.htcondor import HTCondorParser
 
 
 LOGGER_ID = 'parser'
@@ -55,9 +56,11 @@
            'LSF': LSFParser,
            'SGE': SGEParser,
            'SLURM': SlurmParser,
-           'blah' : BlahParser
+           'blah' : BlahParser,
+           'HTCondor': HTCondorParser,
            }
 
+
 class ParserConfigException(Exception):
     '''
     Exception raised when parser is misconfigured.
@@ -154,7 +157,7 @@ def scan_dir(parser, dirpath, reparse, expr, apel_db, processed):
     try:
         log.info('Scanning directory: %s', dirpath)
 
-        for item in os.listdir(dirpath):
+        for item in sorted(os.listdir(dirpath)):
             abs_file = os.path.join(dirpath, item)
             if os.path.isfile(abs_file) and expr.match(item):
                 # first, calculate the hash of the file:

conf/parser.cfg

@@ -29,7 +29,7 @@ enabled = true
 reparse = false
 
 # Batch system specific options.
-# Valid types are LSF, PBS, SGE, SLURM
+# Valid types are LSF, PBS, SGE, SLURM, HTCondor
 type = 
 # Whether to try to parse multi-core details
 parallel = true

requirements.txt

@@ -0,0 +1,7 @@
+MySQL-python
+python-ldap
+iso8601
+dirq
+# Dependencies for testing
+unittest2
+coveralls

schemas/server.sql

@@ -762,13 +762,13 @@ CREATE VIEW VNormalisedSuperSummaries AS
          Infrastructure,
          NodeCount,
          Processors,
-         EarliestEndTime,
-         LatestEndTime,
+         MIN(EarliestEndTime) AS EarliestEndTime,
+         MAX(LatestEndTime) AS LatestEndTime,
          SUM(WallDuration) AS WallDuration,
          SUM(CpuDuration) AS CpuDuration,
          SUM(NormalisedWallDuration) AS NormalisedWallDuration,
          SUM(NormalisedCpuDuration) AS NormalisedCpuDuration,
-         NumberOfJobs
+         SUM(NumberOfJobs) AS NumberOfJobs
   FROM HybridSuperSummaries,
        Sites AS site,
        DNs AS userdn,

scripts/htcondor_acc.sh

@@ -0,0 +1,38 @@
+#! /bin/sh
+
+CONDOR_LOCATION=/usr
+OUTPUT_LOCATION=/var/log/accounting
+
+# Find all the history files modified in the last two months
+# (there can be more than one, if the CE submits to several schedds)
+HISTORY_FILES=$(find /var/lib/condor/spool/ -name history\* -mtime -62)
+
+# Create a temporary accounting file name
+NOW=$(date +"%Y%m%dT%H%M%S")
+OUTPUT_FILE=$OUTPUT_LOCATION/accounting.$NOW
+
+# Build the filter for the history command
+CONSTR="(JobStartDate>0)&&(CompletionDate>`date +%s -d "01 Jan 2014"`)"
+
+# Populate the temporary file
+for HF in $HISTORY_FILES
+do
+  $CONDOR_LOCATION/bin/condor_history -file $HF -constraint $CONSTR \
+    -format "%s|" GlobalJobId \
+    -format "%s|" Owner \
+    -format "%d|" RemoteWallClockTime \
+    -format "%d|" RemoteUserCpu \
+    -format "%d|" RemoteSysCpu \
+    -format "%d|" JobStartDate \
+    -format "%d|" EnteredCurrentStatus \
+    -format "%d|" ResidentSetSize_RAW \
+    -format "%d|" ImageSize_RAW \
+    -format "%d|" RequestCpus \
+    -format "\n" EMPTY >> $OUTPUT_FILE
+done
+
+# Invoke the parser
+/usr/bin/apelparser --config /etc/apel/parser.cfg
+
+# Cleanup
+/bin/find $OUTPUT_LOCATION -name accounting.\* -mtime +30 -exec /bin/rm {} \;

scripts/migrate_apel.py

@@ -57,6 +57,21 @@
 CALLPROC_STMT = """CALL InsertJobRecord(%s, %s, %s, 'None', %s, %s, %s, %s, %s, %s, %s, 
                   %s, %s, NULL, NULL, %s, %s, %s, %s, %s, %s, %s, %s, %s)"""
 
+DUPLICATES_JOIN = """ FROM JobRecords AS t 
+                      LEFT JOIN MachineNames as m 
+		      on (t.MachineNameID = m.id) 
+		      INNER JOIN (SELECT LocalJobId, 
+		                  EndTime 
+				  FROM JobRecords 
+				  LEFT JOIN MachineNames 
+				  on (JobRecords.MachineNameID = MachineNames.id) 
+				  WHERE MachineNames.name != 'MachineName' ) 
+				  AS u 
+                      ON (m.name = 'MachineName' AND t.LocalJobId = u.LocalJobId AND t.EndTime = u.EndTime); """
+
+COUNT_DUPLICATES_STMT = "SELECT count(*) " + DUPLICATES_JOIN
+
+DELETE_DUPLICATES_STMT = "DELETE t " + DUPLICATES_JOIN
 
 def parse_timestamp(string, fmt="%Y-%m-%dT%H:%M:%SZ"):
     '''
@@ -188,6 +203,23 @@ def delete_old_records(db, cutoff):
     c.execute(REMOVE_PROC)
     db.commit()
 
+def delete_duplicates(db):
+    '''
+    Delete all records that have been migrated but are duplicates of records
+    that are in the database already
+    '''
+    c = db.cursor()
+
+    c.execute(COUNT_DUPLICATES_STMT)
+    num_duplicates = c.fetchone()[0]
+    sys.stdout.write('    Found %d duplicate entries\n' % num_duplicates)
+
+    sys.stdout.write('    Deleting duplicates\n')
+    c.execute(DELETE_DUPLICATES_STMT)
+    num_deleted = db.affected_rows()
+    sys.stdout.write('    Deleted %d duplicate entries\n\n\n' % num_deleted)
+
+    db.commit()
 
 def main():
     '''
@@ -235,6 +267,10 @@ def main():
     sys.stdout.write("\n    Deleting all records older than %s\n" % cutoff)
     sys.stdout.write("    from %s:%s ...\n" % (host2, dbname2))
     delete_old_records(db2, cutoff)
+    sys.stdout.write("    Done.\n\n\n")
+
+    # delete duplicates
+    delete_duplicates(db2)
     sys.stdout.write("    Complete.\n\n\n")