Merge pull request #200 from /issues/198/move-to-cfs

Move to cfs

CONTRIBUTING.md

@@ -173,5 +173,5 @@ Then you can simply run `./run_master.sh`; however, there are many useful option
 
 As the master script is running, all the error messages will be printed out in real time if you have set `-v`. You can also go to the web interface to check you result:
 
-https://portal.nersc.gov/projecta/lsst/descqa/v2/?run=all
+https://portal.nersc.gov/cfs/lsst/descqa/v2/?run=all
 

README.md

@@ -7,7 +7,7 @@
 
 This repository contains the DESCQA framework that validates simulated galaxy catalogs. For more information about this framework, please check out the [DESCQA paper](https://arxiv.org/abs/1709.09665).
 
-A [web interface](https://portal.nersc.gov/projecta/lsst/descqa/) hosted on NERSC displays recent validation results from the DESCQA framework.
+A [web interface](https://portal.nersc.gov/cfs/lsst/descqa/) hosted on NERSC displays recent validation results from the DESCQA framework.
 
 **! Important !** Starting from DESCQA v2 (current version), we have separated the configurations and readers of catalogs from DESCQA and moved them to a standalone repo, the [GCRCatalogs](https://github.com/LSSTDESC/gcr-catalogs) repo. We have also changed much of the validation tests. If you are looking for the catalogs and tests in DESCQA v1 (as presented in the [companion paper](https://arxiv.org/abs/1709.09665)), please see the [v1 subdiectory](v1).
 
@@ -20,7 +20,7 @@ A [web interface](https://portal.nersc.gov/projecta/lsst/descqa/) hosted on NERS
 
 3. Now that you are able to make some plots, think about how to "validate" the catalogs (i.e., are there any observation/theory data that can be plotted on the same figure for comparison? How to decide whether a catalog is satisfactory?)
 
-4. Now we can integrate your work into the [DESCQA web interface](https://portal.nersc.gov/projecta/lsst/descqa/v2/)! This step is slightly more involved, but you can follow [the instruction here](CONTRIBUTING.md).
+4. Now we can integrate your work into the [DESCQA web interface](https://portal.nersc.gov/cfs/lsst/descqa/v2/)! This step is slightly more involved, but you can follow [the instruction here](CONTRIBUTING.md).
 
 
 

descqa/ColorDistribution.py

@@ -44,8 +44,6 @@ class ColorDistribution(BaseValidationTest):
     summary_output_file = 'summary.txt'
     plot_pdf_file = 'plot_pdf.png'
     plot_cdf_file = 'plot_cdf.png'
-    sdss_path = '/global/projecta/projectdirs/lsst/groups/CS/descqa/data/rongpu/SpecPhoto_sdss_mgs_extinction_corrected.fits'
-    deep2_path = '/global/projecta/projectdirs/lsst/groups/CS/descqa/data/rongpu/DEEP2_uniq_Terapix_Subaru_trimmed_wights_added.fits'
 
     def __init__(self, **kwargs): # pylint: disable=W0231
 
@@ -77,6 +75,9 @@ def __init__(self, **kwargs): # pylint: disable=W0231
         self.binsize = self.bins[1] - self.bins[0]
 
         # Load validation catalog and define catalog-specific properties
+        self.sdss_path = os.path.join(self.external_data_dir, 'rongpu', 'SpecPhoto_sdss_mgs_extinction_corrected.fits')
+        self.deep2_path = os.path.join(self.external_data_dir, 'rongpu', 'DEEP2_uniq_Terapix_Subaru_trimmed_wights_added.fits')
+
         if self.validation_catalog == 'SDSS':
             obs_path = self.sdss_path
             obscat = Table.read(obs_path)

descqa/DensityVersusSkyPosition.py

@@ -34,7 +34,7 @@ def __init__(self,**kwargs): # pylint: disable=W0231
 
         self.kwargs = kwargs
         self.test_name = kwargs['test_name']
-        self.validation_path = kwargs['validation_map_filename']
+        self.validation_path = os.path.join(self.external_data_dir, kwargs['validation_map_filename'])
         self.nside = kwargs['nside']
         self.validation_data = hp.ud_grade(hp.read_map(self.validation_path), nside_out=self.nside)
         self.xlabel = kwargs['xlabel']

descqa/ImgPkTest.py

@@ -37,6 +37,7 @@ def __init__(self, raft=None, rebinning=None, validation_data_path=None,
         if validation_data_path is None:
             self.validation_data = None
         else:
+            validation_data_path = os.path.join(self.external_data_dir, validation_data_path)
             self.validation_data = Table.read(validation_data_path)
         self.validation_data_label = validation_data_label
         self.pixel_scale = pixel_scale

descqa/base.py

@@ -74,6 +74,7 @@ class BaseValidationTest(object):
     """
 
     data_dir = os.path.join(os.path.dirname(__file__), 'data')
+    external_data_dir = "/global/cfs/cdirs/lsst/groups/CS/descqa/data"
 
     def __init__(self, **kwargs):
         pass

descqa/configs/DensityVersusExtinction.yaml

@@ -1,6 +1,6 @@
 subclass_name: DensityVersusSkyPosition.DensityVersusSkyPosition
 test_name: Density versus sky position
-validation_map_filename: /global/projecta/projectdirs/lsst/groups/LSS/DC1/systematics/lambda_sfd_ebv.fits
+validation_map_filename: DensityVersusExtinction/lambda_sfd_ebv.fits
 nside: 1024
 xlabel: 'E(B-V)'
 description: Variation of mean density of detected objects as a function of extinction (or other map)

descqa/configs/image_power_spec.yaml

@@ -2,4 +2,4 @@ subclass_name: ImgPkTest.ImgPkTest
 description: 'Compute power spectrum of the background. It uses validation data from run #9 in DC2-production issue #140 (PhoSim full background model with sources in it)'
 included_by_default: true
 rebinning: 16
-validation_data_path: /global/projecta/projectdirs/lsst/groups/SSim/DC2/img_pk/reference_w_sources_R22.fits.gz
+validation_data_path: image_power_spec/reference_w_sources_R22.fits.gz

descqarun/config.py

@@ -1,2 +1,2 @@
 __all__ = ['base_url']
-base_url = 'https://portal.nersc.gov/projecta/lsst/descqa/v2/'
+base_url = 'https://portal.nersc.gov/cfs/lsst/descqa/v2/'

descqaweb/config.py

@@ -2,7 +2,7 @@
 
 __all__ = ['site_title', 'root_dir', 'general_info', 'static_dir', 'run_per_page', 'logo_filename', 'github_url', 'months_to_search']
 
-root_dir = '/global/projecta/projectdirs/lsst/groups/CS/descqa/run/v2'
+root_dir = '/global/cfs/cdirs/lsst/groups/CS/descqa/run/v2'
 site_title = 'DESCQA (v2): LSST DESC Quality Assurance for Galaxy Catalogs'
 
 run_per_page = 20
@@ -13,7 +13,7 @@
 github_url = 'https://github.com/lsstdesc/descqa'
 
 general_info = '''
-This is DESCQA v2. You can also visit the previous version, <a class="everblue" href="https://portal.nersc.gov/projecta/lsst/descqa/v1/">DESCQA v1</a>.
+This is DESCQA v2. You can also visit the previous version, <a class="everblue" href="https://portal.nersc.gov/cfs/lsst/descqa/v1/">DESCQA v1</a>.
 <br><br>
 The DESCQA framework executes validation tests on mock galaxy catalogs.
 These tests and catalogs are contributed by LSST DESC collaborators.

run_master.sh

@@ -10,7 +10,7 @@ set -e
 PYTHON="/global/common/software/lsst/common/miniconda/current/envs/desc/bin/python"
 
 # set output directory
-OUTPUTDIR="/global/projecta/projectdirs/lsst/groups/CS/descqa/run/v2"
+OUTPUTDIR="/global/cfs/cdirs/lsst/groups/CS/descqa/run/v2"
 
 # to allow wildcards in arguments go to master.py
 set -o noglob

run_master_v1.sh

@@ -8,12 +8,12 @@ set -e
 
 # activate python env
 PYTHON="/global/common/cori/contrib/lsst/apps/anaconda/py2-envs/DESCQA/bin/python"
-export KCORRECT_DIR="/global/projecta/projectdirs/lsst/groups/CS/descqa/lib/kcorrect"
+export KCORRECT_DIR="/global/cfs/cdirs/lsst/groups/CS/descqa/lib/kcorrect"
 export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$KCORRECT_DIR/lib"
 
 # set output directory
-OUTPUTDIR="/global/projecta/projectdirs/lsst/groups/CS/descqa/run/v1"
-URL="https://portal.nersc.gov/projecta/lsst/descqa/v1/"
+OUTPUTDIR="/global/cfs/cdirs/lsst/groups/CS/descqa/run/v1"
+URL="https://portal.nersc.gov/cfs/lsst/descqa/v1/"
 
 # to allow wildcards in arguments go to master.py
 set -o noglob

v1/GCRCatalogs/YaleCAMGalaxyCatalog.py

@@ -36,7 +36,6 @@ def __init__(self, **kwargs):
 
         # set file type and location
         self.type_ext = 'hdf5'
-        self.root_path = '/global/project/projectdirs/lsst/descqa/'
 
         # set fixed properties
         self.lightcone = False
@@ -82,7 +81,7 @@ def load(self, fn='yale_cam_age_matching_LiWhite_2009_z0.0.hdf5'):
         Parameters
         ----------
         fn : string
-            filename of mock catalog located at self.root_path
+            filename of mock catalog
         """
 
         #extract mock parameters from filename

v1/GCRCatalogs/config.py

@@ -1,2 +1,2 @@
 __all__ = ['base_catalog_dir']
-base_catalog_dir = '/global/projecta/projectdirs/lsst/groups/CS/descqa/catalog'
+base_catalog_dir = '/global/cfs/cdirs/lsst/groups/CS/descqa/catalog'

v1/descqa/ColorDistributionTest.py

@@ -18,7 +18,7 @@
 plot_pdf_file = 'plot_pdf.png'
 plot_cdf_file = 'plot_cdf.png'
 plot_pdf_cdf_file = 'plot_g-r_pdf_cdf.pdf'
-data_dir = '/global/projecta/projectdirs/lsst/groups/CS/descqa/data/rongpu/'
+data_dir = '/global/cfs/cdirs/lsst/groups/CS/descqa/data/rongpu/'
 data_name = 'SDSS'
 
 limiting_band_name = 'SDSS_r:rest:'