diff --git a/.gitignore b/.gitignore index 946d05521..bd62c4b79 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,10 @@ # Ignore the database file and misc *.db +*.sqlite *.pyc .idea .vscode .pytest_cache .ds_store -bdnyc/ \ No newline at end of file +*.code-workspace + diff --git a/data/Versions.json b/data/Versions.json index 9ad0796a2..9df07db7e 100644 --- a/data/Versions.json +++ b/data/Versions.json @@ -57,6 +57,6 @@ "version": "latest", "start_date": "2023-12-12", "end_date": null, - "description": "Version in development" + "description": "Version in development" } ] \ No newline at end of file diff --git a/data/cwise_j000021.45-481314.9.json b/data/cwise_j000021.45-481314.9.json new file mode 100644 index 000000000..d79e1a87a --- /dev/null +++ b/data/cwise_j000021.45-481314.9.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J000021.45-481314.9", + "ra": 0.0893808, + "dec": -48.2208077, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0000-4813" + }, + { + "other_name": "CWISE J000021.45-481314.9" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j002029.72-153527.5.json b/data/cwise_j002029.72-153527.5.json new file mode 100644 index 000000000..676b23f61 --- /dev/null +++ b/data/cwise_j002029.72-153527.5.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J002029.72-153527.5", + "ra": 5.1238388, + "dec": -15.5909815, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0020-1535" + }, + { + "other_name": "CWISE J002029.72-153527.5" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j002101.45-334631.5.json b/data/cwise_j002101.45-334631.5.json new file mode 100644 index 000000000..130530cda --- /dev/null +++ b/data/cwise_j002101.45-334631.5.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J002101.45-334631.5", + "ra": 5.2560473, + "dec": -33.7754218, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0021-3346" + }, + { + "other_name": "CWISE J002101.45-334631.5" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j002159.02-451434.4.json b/data/cwise_j002159.02-451434.4.json new file mode 100644 index 000000000..f2daec136 --- /dev/null +++ b/data/cwise_j002159.02-451434.4.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J002159.02-451434.4", + "ra": 5.4959547, + "dec": -45.2428987, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0021-4514" + }, + { + "other_name": "CWISE J002159.02-451434.4" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j002414.30-403053.7.json b/data/cwise_j002414.30-403053.7.json new file mode 100644 index 000000000..2c80b2394 --- /dev/null +++ b/data/cwise_j002414.30-403053.7.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J002414.30-403053.7", + "ra": 6.0596207, + "dec": -40.5149259, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0024-4030" + }, + { + "other_name": "CWISE J002414.30-403053.7" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j002658.73-260859.6.json b/data/cwise_j002658.73-260859.6.json new file mode 100644 index 000000000..e955e40c9 --- /dev/null +++ b/data/cwise_j002658.73-260859.6.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J002658.73-260859.6", + "ra": 6.7447362, + "dec": -26.1499067, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0026-2608" + }, + { + "other_name": "CWISE J002658.73-260859.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j004218.67+291730.6.json b/data/cwise_j004218.67+291730.6.json new file mode 100644 index 000000000..3de7e3c0b --- /dev/null +++ b/data/cwise_j004218.67+291730.6.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J004218.67+291730.6", + "ra": 10.5777904, + "dec": 29.2918566, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0042+2917" + }, + { + "other_name": "CWISE J004218.67+291730.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j004945.38+423619.3.json b/data/cwise_j004945.38+423619.3.json new file mode 100644 index 000000000..857156570 --- /dev/null +++ b/data/cwise_j004945.38+423619.3.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J004945.38+423619.3", + "ra": 12.439107, + "dec": 42.6053806, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0049+4236" + }, + { + "other_name": "CWISE J004945.38+423619.3" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j005635.48-240401.9.json b/data/cwise_j005635.48-240401.9.json new file mode 100644 index 000000000..c48c8e4f9 --- /dev/null +++ b/data/cwise_j005635.48-240401.9.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J005635.48-240401.9", + "ra": 14.1478506, + "dec": -24.0672083, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0056-2404" + }, + { + "other_name": "CWISE J005635.48-240401.9" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j010424.50+133949.8.json b/data/cwise_j010424.50+133949.8.json new file mode 100644 index 000000000..801ee5170 --- /dev/null +++ b/data/cwise_j010424.50+133949.8.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J010424.50+133949.8", + "ra": 16.1020944, + "dec": 13.6638352, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0104+1339" + }, + { + "other_name": "CWISE J010424.50+133949.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j011003.61-592650.8.json b/data/cwise_j011003.61-592650.8.json new file mode 100644 index 000000000..e9b9ef794 --- /dev/null +++ b/data/cwise_j011003.61-592650.8.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J011003.61-592650.8", + "ra": 17.5150429, + "dec": -59.4474481, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0110-5926" + }, + { + "other_name": "CWISE J011003.61-592650.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j012715.52-300246.9.json b/data/cwise_j012715.52-300246.9.json new file mode 100644 index 000000000..12cfbad71 --- /dev/null +++ b/data/cwise_j012715.52-300246.9.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J012715.52-300246.9", + "ra": 21.8146967, + "dec": -30.0463841, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0127-3002" + }, + { + "other_name": "CWISE J012715.52-300246.9" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j013719.04+342837.6.json b/data/cwise_j013719.04+342837.6.json new file mode 100644 index 000000000..a09eb9b9c --- /dev/null +++ b/data/cwise_j013719.04+342837.6.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J013719.04+342837.6", + "ra": 24.3293435, + "dec": 34.4771182, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0137+3428" + }, + { + "other_name": "CWISE J013719.04+342837.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j015905.58+105551.5.json b/data/cwise_j015905.58+105551.5.json new file mode 100644 index 000000000..30b052924 --- /dev/null +++ b/data/cwise_j015905.58+105551.5.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J015905.58+105551.5", + "ra": 29.773276, + "dec": 10.9309815, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0159+1055" + }, + { + "other_name": "CWISE J015905.58+105551.5" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j020538.20+594452.2.json b/data/cwise_j020538.20+594452.2.json new file mode 100644 index 000000000..c653a2423 --- /dev/null +++ b/data/cwise_j020538.20+594452.2.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J020538.20+594452.2", + "ra": 31.40993946, + "dec": 59.74890954, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0205+5944" + }, + { + "other_name": "CWISE J020538.20+594452.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j022454.80+152629.5.json b/data/cwise_j022454.80+152629.5.json new file mode 100644 index 000000000..1a9fca912 --- /dev/null +++ b/data/cwise_j022454.80+152629.5.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J022454.80+152629.5", + "ra": 36.2283563, + "dec": 15.441534, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0224+1526*" + }, + { + "other_name": "CWISE J022454.80+152629.5" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j022737.75+083008.8.json b/data/cwise_j022737.75+083008.8.json new file mode 100644 index 000000000..b6f8a8972 --- /dev/null +++ b/data/cwise_j022737.75+083008.8.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J022737.75+083008.8", + "ra": 36.9072967, + "dec": 8.5024629, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0227+0830" + }, + { + "other_name": "CWISE J022737.75+083008.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j025645.16-335008.9.json b/data/cwise_j025645.16-335008.9.json new file mode 100644 index 000000000..f0261db06 --- /dev/null +++ b/data/cwise_j025645.16-335008.9.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J025645.16-335008.9", + "ra": 44.188202, + "dec": -33.8358082, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0256-3350" + }, + { + "other_name": "CWISE J025645.16-335008.9" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j030005.73-062218.6.json b/data/cwise_j030005.73-062218.6.json new file mode 100644 index 000000000..685a2ca83 --- /dev/null +++ b/data/cwise_j030005.73-062218.6.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J030005.73-062218.6", + "ra": 45.0238923, + "dec": -6.371848, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0300-0622" + }, + { + "other_name": "CWISE J030005.73-062218.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j032853.32-112332.6.json b/data/cwise_j032853.32-112332.6.json new file mode 100644 index 000000000..41db3dc03 --- /dev/null +++ b/data/cwise_j032853.32-112332.6.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J032853.32-112332.6", + "ra": 52.2221958, + "dec": -11.3924108, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0328-1123" + }, + { + "other_name": "CWISE J032853.32-112332.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j055515.83+510514.0.json b/data/cwise_j055515.83+510514.0.json new file mode 100644 index 000000000..df388498e --- /dev/null +++ b/data/cwise_j055515.83+510514.0.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J055515.83+510514.0", + "ra": 88.8159856, + "dec": 51.0872307, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0555+5105" + }, + { + "other_name": "CWISE J055515.83+510514.0" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j055909.00-384219.8.json b/data/cwise_j055909.00-384219.8.json new file mode 100644 index 000000000..27037ec53 --- /dev/null +++ b/data/cwise_j055909.00-384219.8.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J055909.00-384219.8", + "ra": 89.787502, + "dec": -38.7055027, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0559-3842" + }, + { + "other_name": "CWISE J055909.00-384219.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j060202.17-462447.8.json b/data/cwise_j060202.17-462447.8.json new file mode 100644 index 000000000..d8108901c --- /dev/null +++ b/data/cwise_j060202.17-462447.8.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J060202.17-462447.8", + "ra": 90.509052, + "dec": -46.4132863, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0602-4624" + }, + { + "other_name": "CWISE J060202.17-462447.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j062000.01+344641.3.json b/data/cwise_j062000.01+344641.3.json new file mode 100644 index 000000000..ec485010d --- /dev/null +++ b/data/cwise_j062000.01+344641.3.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J062000.01+344641.3", + "ra": 95.0000552, + "dec": 34.7781567, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0620+3446" + }, + { + "other_name": "CWISE J062000.01+344641.3" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j062648.96+594129.2.json b/data/cwise_j062648.96+594129.2.json new file mode 100644 index 000000000..d6d971a68 --- /dev/null +++ b/data/cwise_j062648.96+594129.2.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J062648.96+594129.2", + "ra": 96.7040383, + "dec": 59.6914571, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0626+5941" + }, + { + "other_name": "CWISE J062648.96+594129.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j062727.34-002826.8.json b/data/cwise_j062727.34-002826.8.json new file mode 100644 index 000000000..3588662b2 --- /dev/null +++ b/data/cwise_j062727.34-002826.8.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J062727.34-002826.8", + "ra": 96.8639415, + "dec": -0.474135, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0627-0028A" + }, + { + "other_name": "CWISE J062727.34-002826.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j062909.21+395701.6.json b/data/cwise_j062909.21+395701.6.json new file mode 100644 index 000000000..498af5976 --- /dev/null +++ b/data/cwise_j062909.21+395701.6.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J062909.21+395701.6", + "ra": 97.2884074, + "dec": 39.950463, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0629+3957" + }, + { + "other_name": "CWISE J062909.21+395701.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j065752.45+163350.2.json b/data/cwise_j065752.45+163350.2.json new file mode 100644 index 000000000..e71656093 --- /dev/null +++ b/data/cwise_j065752.45+163350.2.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J065752.45+163350.2", + "ra": 104.4685682, + "dec": 16.5639658, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0657+1633" + }, + { + "other_name": "CWISE J065752.45+163350.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j073831.31+525453.7.json b/data/cwise_j073831.31+525453.7.json new file mode 100644 index 000000000..f27e65755 --- /dev/null +++ b/data/cwise_j073831.31+525453.7.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J073831.31+525453.7", + "ra": 114.6304785, + "dec": 52.9149251, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0738+5254" + }, + { + "other_name": "CWISE J073831.31+525453.7" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j080912.95+741704.3.json b/data/cwise_j080912.95+741704.3.json new file mode 100644 index 000000000..50c789409 --- /dev/null +++ b/data/cwise_j080912.95+741704.3.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J080912.95+741704.3", + "ra": 122.3039975, + "dec": 74.2845438, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0809+7417" + }, + { + "other_name": "CWISE J080912.95+741704.3" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j085131.24-603056.2.json b/data/cwise_j085131.24-603056.2.json new file mode 100644 index 000000000..60025a2da --- /dev/null +++ b/data/cwise_j085131.24-603056.2.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J085131.24-603056.2", + "ra": 132.8801846, + "dec": -60.5156128, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0851-6030" + }, + { + "other_name": "CWISE J085131.24-603056.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j091558.53+254713.0.json b/data/cwise_j091558.53+254713.0.json new file mode 100644 index 000000000..4109a8c7a --- /dev/null +++ b/data/cwise_j091558.53+254713.0.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J091558.53+254713.0", + "ra": 138.9939062, + "dec": 25.7869589, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0915+2547" + }, + { + "other_name": "CWISE J091558.53+254713.0" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j091902.55-574837.3.json b/data/cwise_j091902.55-574837.3.json new file mode 100644 index 000000000..6b2912691 --- /dev/null +++ b/data/cwise_j091902.55-574837.3.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J091902.55-574837.3", + "ra": 139.7606504, + "dec": -57.8103837, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0919-5748" + }, + { + "other_name": "CWISE J091902.55-574837.3" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j094352.22+335639.1.json b/data/cwise_j094352.22+335639.1.json new file mode 100644 index 000000000..4616b7ca6 --- /dev/null +++ b/data/cwise_j094352.22+335639.1.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J094352.22+335639.1", + "ra": 145.9676104, + "dec": 33.9442094, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0943+3356" + }, + { + "other_name": "CWISE J094352.22+335639.1" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j095259.54-505418.9.json b/data/cwise_j095259.54-505418.9.json new file mode 100644 index 000000000..b9bb92901 --- /dev/null +++ b/data/cwise_j095259.54-505418.9.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J095259.54-505418.9", + "ra": 148.2480941, + "dec": -50.905256, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 0952-5054" + }, + { + "other_name": "CWISE J095259.54-505418.9" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j101017.43-242300.4.json b/data/cwise_j101017.43-242300.4.json new file mode 100644 index 000000000..d59e2597d --- /dev/null +++ b/data/cwise_j101017.43-242300.4.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J101017.43-242300.4", + "ra": 152.5726326, + "dec": -24.3834649, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1010-2423" + }, + { + "other_name": "CWISE J101017.43-242300.4" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j101317.51-010313.6.json b/data/cwise_j101317.51-010313.6.json new file mode 100644 index 000000000..9cbed6b15 --- /dev/null +++ b/data/cwise_j101317.51-010313.6.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J101317.51-010313.6", + "ra": 153.3229961, + "dec": -1.0537976, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1013-0103" + }, + { + "other_name": "CWISE J101317.51-010313.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j101523.92-111539.6.json b/data/cwise_j101523.92-111539.6.json new file mode 100644 index 000000000..ca1fb8ae1 --- /dev/null +++ b/data/cwise_j101523.92-111539.6.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J101523.92-111539.6", + "ra": 153.849691, + "dec": -11.2610035, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1015-1115" + }, + { + "other_name": "CWISE J101523.92-111539.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j103734.29-050749.8.json b/data/cwise_j103734.29-050749.8.json new file mode 100644 index 000000000..cdb162fbe --- /dev/null +++ b/data/cwise_j103734.29-050749.8.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J103734.29-050749.8", + "ra": 159.3929093, + "dec": -5.1305155, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1037-0507" + }, + { + "other_name": "CWISE J103734.29-050749.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j104053.42-355029.7.json b/data/cwise_j104053.42-355029.7.json new file mode 100644 index 000000000..8891640db --- /dev/null +++ b/data/cwise_j104053.42-355029.7.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J104053.42-355029.7", + "ra": 160.2226104, + "dec": -35.8415843, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1040-3550" + }, + { + "other_name": "CWISE J104053.42-355029.7" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j115229.05-351105.9.json b/data/cwise_j115229.05-351105.9.json new file mode 100644 index 000000000..9b2ec85b5 --- /dev/null +++ b/data/cwise_j115229.05-351105.9.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J115229.05-351105.9", + "ra": 178.1210618, + "dec": -35.1849737, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1152-3511" + }, + { + "other_name": "CWISE J115229.05-351105.9" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j124033.48+460544.7.json b/data/cwise_j124033.48+460544.7.json new file mode 100644 index 000000000..3838f23b4 --- /dev/null +++ b/data/cwise_j124033.48+460544.7.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J124033.48+460544.7", + "ra": 190.1395201, + "dec": 46.0957506, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1240+4605" + }, + { + "other_name": "CWISE J124033.48+460544.7" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j125858.20+001133.2.json b/data/cwise_j125858.20+001133.2.json new file mode 100644 index 000000000..2aa99ad16 --- /dev/null +++ b/data/cwise_j125858.20+001133.2.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J125858.20+001133.2", + "ra": 194.7425134, + "dec": 0.1925822, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1258+0011" + }, + { + "other_name": "CWISE J125858.20+001133.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j130329.90+512754.0.json b/data/cwise_j130329.90+512754.0.json new file mode 100644 index 000000000..21027628d --- /dev/null +++ b/data/cwise_j130329.90+512754.0.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J130329.90+512754.0", + "ra": 195.8745856, + "dec": 51.4650208, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1303+5127" + }, + { + "other_name": "CWISE J130329.90+512754.0" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j130446.64-120023.6.json b/data/cwise_j130446.64-120023.6.json new file mode 100644 index 000000000..f133da069 --- /dev/null +++ b/data/cwise_j130446.64-120023.6.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J130446.64-120023.6", + "ra": 196.1944474, + "dec": -12.00636689, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1304-1200" + }, + { + "other_name": "CWISE J130446.64-120023.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j131355.15+223005.6.json b/data/cwise_j131355.15+223005.6.json new file mode 100644 index 000000000..a86894775 --- /dev/null +++ b/data/cwise_j131355.15+223005.6.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J131355.15+223005.6", + "ra": 198.4798067, + "dec": 22.5015715, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1313+2230" + }, + { + "other_name": "CWISE J131355.15+223005.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j132539.70+022309.4.json b/data/cwise_j132539.70+022309.4.json new file mode 100644 index 000000000..713b05053 --- /dev/null +++ b/data/cwise_j132539.70+022309.4.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J132539.70+022309.4", + "ra": 201.4154343, + "dec": 2.3859476, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1325+0223" + }, + { + "other_name": "CWISE J132539.70+022309.4" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j132857.58-063747.4.json b/data/cwise_j132857.58-063747.4.json new file mode 100644 index 000000000..8838331c5 --- /dev/null +++ b/data/cwise_j132857.58-063747.4.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J132857.58-063747.4", + "ra": 202.2399559, + "dec": -6.6298609, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1328-0637" + }, + { + "other_name": "CWISE J132857.58-063747.4" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j133211.93-374837.9.json b/data/cwise_j133211.93-374837.9.json new file mode 100644 index 000000000..605287722 --- /dev/null +++ b/data/cwise_j133211.93-374837.9.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J133211.93-374837.9", + "ra": 203.049724, + "dec": -37.8105289, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1332-3748" + }, + { + "other_name": "CWISE J133211.93-374837.9" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j133427.70-273053.1.json b/data/cwise_j133427.70-273053.1.json new file mode 100644 index 000000000..6377fd1a2 --- /dev/null +++ b/data/cwise_j133427.70-273053.1.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J133427.70-273053.1", + "ra": 203.6154266, + "dec": -27.5147656, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1334-2730" + }, + { + "other_name": "CWISE J133427.70-273053.1" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j141737.21+041847.2.json b/data/cwise_j141737.21+041847.2.json new file mode 100644 index 000000000..bdc58c9e3 --- /dev/null +++ b/data/cwise_j141737.21+041847.2.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J141737.21+041847.2", + "ra": 214.4050608, + "dec": 4.3131162, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1417+0418" + }, + { + "other_name": "CWISE J141737.21+041847.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j143935.94-804851.5.json b/data/cwise_j143935.94-804851.5.json new file mode 100644 index 000000000..93cf82d05 --- /dev/null +++ b/data/cwise_j143935.94-804851.5.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J143935.94-804851.5", + "ra": 219.8997509, + "dec": -80.8143328, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1439-8048" + }, + { + "other_name": "CWISE J143935.94-804851.5" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j143951.51+260000.3.json b/data/cwise_j143951.51+260000.3.json new file mode 100644 index 000000000..44bb7d7be --- /dev/null +++ b/data/cwise_j143951.51+260000.3.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J143951.51+260000.3", + "ra": 219.964641, + "dec": 26.0000984, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1439+2600" + }, + { + "other_name": "CWISE J143951.51+260000.3" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j144057.87+380432.8.json b/data/cwise_j144057.87+380432.8.json new file mode 100644 index 000000000..b8128b4dd --- /dev/null +++ b/data/cwise_j144057.87+380432.8.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J144057.87+380432.8", + "ra": 220.2411279, + "dec": 38.0757925, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1440+3804" + }, + { + "other_name": "CWISE J144057.87+380432.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j151939.68-485820.4.json b/data/cwise_j151939.68-485820.4.json new file mode 100644 index 000000000..1536af124 --- /dev/null +++ b/data/cwise_j151939.68-485820.4.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J151939.68-485820.4", + "ra": 229.9153563, + "dec": -48.972359, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1519-4858" + }, + { + "other_name": "CWISE J151939.68-485820.4" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j152742.44-121528.2.json b/data/cwise_j152742.44-121528.2.json new file mode 100644 index 000000000..1c5e5e016 --- /dev/null +++ b/data/cwise_j152742.44-121528.2.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J152742.44-121528.2", + "ra": 231.9268706, + "dec": -12.2578539, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1527-1215" + }, + { + "other_name": "CWISE J152742.44-121528.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j153910.07+722757.2.json b/data/cwise_j153910.07+722757.2.json new file mode 100644 index 000000000..93ff3055a --- /dev/null +++ b/data/cwise_j153910.07+722757.2.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J153910.07+722757.2", + "ra": 234.7919959, + "dec": 72.4658937, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1539+7227" + }, + { + "other_name": "CWISE J153910.07+722757.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j160654.19-103214.7.json b/data/cwise_j160654.19-103214.7.json new file mode 100644 index 000000000..cd8036bb7 --- /dev/null +++ b/data/cwise_j160654.19-103214.7.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J160654.19-103214.7", + "ra": 241.7258072, + "dec": -10.5374259, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1606-1032" + }, + { + "other_name": "CWISE J160654.19-103214.7" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j162511.27+774946.8.json b/data/cwise_j162511.27+774946.8.json new file mode 100644 index 000000000..1764e2e7a --- /dev/null +++ b/data/cwise_j162511.27+774946.8.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J162511.27+774946.8", + "ra": 246.2969812, + "dec": 77.8296817, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1625+7749" + }, + { + "other_name": "CWISE J162511.27+774946.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j165143.63+695259.4.json b/data/cwise_j165143.63+695259.4.json new file mode 100644 index 000000000..591c3064d --- /dev/null +++ b/data/cwise_j165143.63+695259.4.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J165143.63+695259.4", + "ra": 252.9318288, + "dec": 69.8831753, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1651+6952" + }, + { + "other_name": "CWISE J165143.63+695259.4" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j165325.10+490439.7.json b/data/cwise_j165325.10+490439.7.json new file mode 100644 index 000000000..383d0810e --- /dev/null +++ b/data/cwise_j165325.10+490439.7.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J165325.10+490439.7", + "ra": 253.3545821, + "dec": 49.0777014, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1653+4904" + }, + { + "other_name": "CWISE J165325.10+490439.7" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j171308.00+644220.8.json b/data/cwise_j171308.00+644220.8.json new file mode 100644 index 000000000..9817b4580 --- /dev/null +++ b/data/cwise_j171308.00+644220.8.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J171308.00+644220.8", + "ra": 258.2833578, + "dec": 64.7058005, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1713+6442" + }, + { + "other_name": "CWISE J171308.00+644220.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j171700.57+724437.3.json b/data/cwise_j171700.57+724437.3.json new file mode 100644 index 000000000..fe46adb1a --- /dev/null +++ b/data/cwise_j171700.57+724437.3.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J171700.57+724437.3", + "ra": 259.2523353, + "dec": 72.74369143, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1717+7244" + }, + { + "other_name": "CWISE J171700.57+724437.3" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j173859.73+200501.2.json b/data/cwise_j173859.73+200501.2.json new file mode 100644 index 000000000..3a96409a6 --- /dev/null +++ b/data/cwise_j173859.73+200501.2.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J173859.73+200501.2", + "ra": 264.7489139, + "dec": 20.0836743, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1738+2005" + }, + { + "other_name": "CWISE J173859.73+200501.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j174426.85+230355.1.json b/data/cwise_j174426.85+230355.1.json new file mode 100644 index 000000000..55c1ab0f4 --- /dev/null +++ b/data/cwise_j174426.85+230355.1.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J174426.85+230355.1", + "ra": 266.1118811, + "dec": 23.0653259, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1744+2303" + }, + { + "other_name": "CWISE J174426.85+230355.1" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j174509.03+380733.2.json b/data/cwise_j174509.03+380733.2.json new file mode 100644 index 000000000..518468bf6 --- /dev/null +++ b/data/cwise_j174509.03+380733.2.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J174509.03+380733.2", + "ra": 266.2876616, + "dec": 38.1259034, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1745+3807" + }, + { + "other_name": "CWISE J174509.03+380733.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j183207.94-540943.3.json b/data/cwise_j183207.94-540943.3.json index e2c0035f4..e4509b9c1 100644 --- a/data/cwise_j183207.94-540943.3.json +++ b/data/cwise_j183207.94-540943.3.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1832-5409" + }, { "other_name": "CWISE J183207.94-540943.3" } diff --git a/data/cwise_j190909.62+553027.1.json b/data/cwise_j190909.62+553027.1.json new file mode 100644 index 000000000..50d0ac073 --- /dev/null +++ b/data/cwise_j190909.62+553027.1.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J190909.62+553027.1", + "ra": 287.2900976, + "dec": 55.5075306, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1909+5530" + }, + { + "other_name": "CWISE J190909.62+553027.1" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j192425.03-532837.1.json b/data/cwise_j192425.03-532837.1.json new file mode 100644 index 000000000..2759e1541 --- /dev/null +++ b/data/cwise_j192425.03-532837.1.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J192425.03-532837.1", + "ra": 291.1043308, + "dec": -53.4769769, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1924-5328" + }, + { + "other_name": "CWISE J192425.03-532837.1" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j195956.16-644318.1.json b/data/cwise_j195956.16-644318.1.json new file mode 100644 index 000000000..3f88d6a83 --- /dev/null +++ b/data/cwise_j195956.16-644318.1.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J195956.16-644318.1", + "ra": 299.9840118, + "dec": -64.7217168, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 1959-6443" + }, + { + "other_name": "CWISE J195956.16-644318.1" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j200304.70-142228.9.json b/data/cwise_j200304.70-142228.9.json new file mode 100644 index 000000000..039066a8c --- /dev/null +++ b/data/cwise_j200304.70-142228.9.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J200304.70-142228.9", + "ra": 300.7696229, + "dec": -14.3747026, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 2003-1422" + }, + { + "other_name": "CWISE J200304.70-142228.9" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j200813.66-124502.4.json b/data/cwise_j200813.66-124502.4.json new file mode 100644 index 000000000..36adb088e --- /dev/null +++ b/data/cwise_j200813.66-124502.4.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J200813.66-124502.4", + "ra": 302.0569331, + "dec": -12.7506846, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 2008-1245" + }, + { + "other_name": "CWISE J200813.66-124502.4" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j202934.80-791013.1.json b/data/cwise_j202934.80-791013.1.json new file mode 100644 index 000000000..0ac35b85c --- /dev/null +++ b/data/cwise_j202934.80-791013.1.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J202934.80-791013.1", + "ra": 307.3950501, + "dec": -79.17033373, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 2029-7910" + }, + { + "other_name": "CWISE J202934.80-791013.1" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j205247.07+053855.7.json b/data/cwise_j205247.07+053855.7.json new file mode 100644 index 000000000..d7e983671 --- /dev/null +++ b/data/cwise_j205247.07+053855.7.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J205247.07+053855.7", + "ra": 313.196131, + "dec": 5.6488179, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 2052+0538" + }, + { + "other_name": "CWISE J205247.07+053855.7" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j210640.16+250729.0.json b/data/cwise_j210640.16+250729.0.json index f4201c39c..e31f6592e 100644 --- a/data/cwise_j210640.16+250729.0.json +++ b/data/cwise_j210640.16+250729.0.json @@ -2,10 +2,10 @@ "Sources": [ { "source": "CWISE J210640.16+250729.0", - "ra": 316.66736, - "dec": 25.124719444444445, - "epoch": null, - "equinox": null, + "ra": 316.6673638, + "dec": 25.1247223, + "epoch": 2015.4041, + "equinox": "ICRS", "shortname": null, "reference": "Roth", "other_references": null, @@ -24,6 +24,9 @@ } ], "Names": [ + { + "other_name": "CWISE 2106+2507" + }, { "other_name": "CWISE J210640.16+250729.0" } diff --git a/data/cwise_j212342.88+655615.6.json b/data/cwise_j212342.88+655615.6.json new file mode 100644 index 000000000..992bafd38 --- /dev/null +++ b/data/cwise_j212342.88+655615.6.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J212342.88+655615.6", + "ra": 320.9286959, + "dec": 65.9376939, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 2123+6556" + }, + { + "other_name": "CWISE J212342.88+655615.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j214129.80-024623.6.json b/data/cwise_j214129.80-024623.6.json new file mode 100644 index 000000000..a4f1937d4 --- /dev/null +++ b/data/cwise_j214129.80-024623.6.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J214129.80-024623.6", + "ra": 325.3741677, + "dec": -2.7732463, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 2141-0246" + }, + { + "other_name": "CWISE J214129.80-024623.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j214213.99-204659.8.json b/data/cwise_j214213.99-204659.8.json new file mode 100644 index 000000000..f4d3c4a67 --- /dev/null +++ b/data/cwise_j214213.99-204659.8.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J214213.99-204659.8", + "ra": 325.558323, + "dec": -20.7832923, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 2142-2046" + }, + { + "other_name": "CWISE J214213.99-204659.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j221418.15+253432.2.json b/data/cwise_j221418.15+253432.2.json new file mode 100644 index 000000000..9af6909ed --- /dev/null +++ b/data/cwise_j221418.15+253432.2.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J221418.15+253432.2", + "ra": 333.5756522, + "dec": 25.5756384, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 2214+2535" + }, + { + "other_name": "CWISE J221418.15+253432.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j224452.18-362946.0.json b/data/cwise_j224452.18-362946.0.json new file mode 100644 index 000000000..00e6cd9eb --- /dev/null +++ b/data/cwise_j224452.18-362946.0.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J224452.18-362946.0", + "ra": 341.2174357, + "dec": -36.4961287, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 2244-3629" + }, + { + "other_name": "CWISE J224452.18-362946.0" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j225525.22-302320.8.json b/data/cwise_j225525.22-302320.8.json new file mode 100644 index 000000000..a6060c28f --- /dev/null +++ b/data/cwise_j225525.22-302320.8.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J225525.22-302320.8", + "ra": 343.8550923, + "dec": -30.3891146, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 2255-3023" + }, + { + "other_name": "CWISE J225525.22-302320.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j231824.58+052142.3.json b/data/cwise_j231824.58+052142.3.json new file mode 100644 index 000000000..03805bc7e --- /dev/null +++ b/data/cwise_j231824.58+052142.3.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J231824.58+052142.3", + "ra": 349.6024391, + "dec": 5.3617605, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 2318+0521" + }, + { + "other_name": "CWISE J231824.58+052142.3" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j233531.55+014219.6.json b/data/cwise_j233531.55+014219.6.json new file mode 100644 index 000000000..942d92ab2 --- /dev/null +++ b/data/cwise_j233531.55+014219.6.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J233531.55+014219.6", + "ra": 353.8814851, + "dec": 1.7054667, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 2335+0142" + }, + { + "other_name": "CWISE J233531.55+014219.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j235827.96-521813.4.json b/data/cwise_j235827.96-521813.4.json new file mode 100644 index 000000000..8b973bf6b --- /dev/null +++ b/data/cwise_j235827.96-521813.4.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "CWISE J235827.96-521813.4", + "ra": 359.6165101, + "dec": -52.3037451, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": null, + "comments": null + } + ], + "Names": [ + { + "other_name": "CWISE 2358-5218" + }, + { + "other_name": "CWISE J235827.96-521813.4" + } + ] +} \ No newline at end of file diff --git a/data/cwisep_j040351.00-491605.6.json b/data/cwisep_j040351.00-491605.6.json index 222ebc0fc..8a0920ae4 100644 --- a/data/cwisep_j040351.00-491605.6.json +++ b/data/cwisep_j040351.00-491605.6.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE J040351.12-491605.4" + }, { "other_name": "CWISEP J040351.00-491605.6" } diff --git a/data/cwisep_j085938.95+534908.7.json b/data/cwisep_j085938.95+534908.7.json index b0d9b0d89..409a9abe4 100644 --- a/data/cwisep_j085938.95+534908.7.json +++ b/data/cwisep_j085938.95+534908.7.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE J085938.91+534908.4" + }, { "other_name": "CWISEP J085938.95+534908.7" } diff --git a/data/ulas_j103131.49+123736.4.json b/data/ulas_j103131.49+123736.4.json new file mode 100644 index 000000000..02bc4d897 --- /dev/null +++ b/data/ulas_j103131.49+123736.4.json @@ -0,0 +1,23 @@ +{ + "Sources": [ + { + "source": "ULAS J103131.49+123736.4", + "ra": 157.8811199, + "dec": 12.6267923, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Skrz16", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "ULAS 1031+1237" + }, + { + "other_name": "ULAS J103131.49+123736.4" + } + ] +} \ No newline at end of file diff --git a/data/ulas_j134403.78+083951.0.json b/data/ulas_j134403.78+083951.0.json index 594e0c29d..cb06fb469 100644 --- a/data/ulas_j134403.78+083951.0.json +++ b/data/ulas_j134403.78+083951.0.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "SDSS J134403.83+083950.9" + }, { "other_name": "ULAS J134403.78+083951.0" } diff --git a/data/wisea_j040702.42+190945.8.json b/data/wisea_j040702.42+190945.8.json index 39e3d19e0..973331fd8 100644 --- a/data/wisea_j040702.42+190945.8.json +++ b/data/wisea_j040702.42+190945.8.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE J040702.50+190944.6" + }, { "other_name": "WISEA J040702.42+190945.8" } diff --git a/scripts/ingests/ingest_BYW_2024.py b/scripts/ingests/ingest_BYW_2024.py new file mode 100644 index 000000000..37b27b972 --- /dev/null +++ b/scripts/ingests/ingest_BYW_2024.py @@ -0,0 +1,119 @@ +from scripts.ingests.ingest_utils import * +from scripts.ingests.utils import * +from astropy.io import ascii + +SAVE_DB = False # save the data files in addition to modifying the .db file +RECREATE_DB = True # recreates the .db file from the data files +# LOAD THE DATABASE +db = load_simpledb("SIMPLE.db", recreatedb=RECREATE_DB) + +# Load Google sheet +sheet_id = "1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs" +link = f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=csv" + +# read the csv data into an astropy table +# ascii.read attempts to read data from local files rather from URLs so using a library like requests helps get data and create object that can be passed to ascii.read +byw_table = ascii.read( + link, + format="csv", + data_start=1, + header_start=0, + guess=False, + fast_reader=False, + delimiter=",", +) + +# print result astropy table +print(byw_table.info) + + +# Loop through each row in byw table and print data: source name ra, dec. +def ingest_all_sources(db): + for row in byw_table[1:90]: # skip the header row - [1:10]runs only first 10 rows + # Print byw source information + print("BYW Source Information:") + + + for col_name in row.colnames: + print(f"{col_name}: {row[col_name]}") + + ingest_source( + db, + source=row["Source"], + reference=row["Reference"], + ra=row["RA"], + dec=row["Dec"], + epoch=row["Epoch"], + equinox=row["Equinox"], + raise_error=True, + search_db=True, + ) + + + # Add a separator between rows for better readability + print("-" * 20) + +#Call sources function +ingest_all_sources(db) + + +#Ingest shortnames as other names to source +#Loop through data +def ingest_all_shortnames(db): + for row in byw_table[1:90]: + + # Print byw source information + print("BYW Source Information:") + + for col_name in row.colnames: + print(f"{col_name}: {row[col_name]}") + + ingest_names(db, + source=row["Source"], + other_name=row["Shortname"] + ) + + print("-" * 20) + +#Call shortnames function +ingest_all_shortnames(db) + + +# Ingested other_ref sources as publications +# Skrzypek et al. 2016, Marocco et al. 2015(Online Catalog), Kirkpatrick et al. 2021 +# Ingest reference name: Skrz16, Maro15, Kirk21 +def add_publication(db): + ingest_publication( + db, doi="10.26093/cds/vizier.35890049", bibcode="2016yCat..35890049S" + ) + + ingest_publication(db, doi="10.1093/mnras/stv530", bibcode="2015MNRAS.449.3651M") + + ingest_publication( + db, doi="10.3847/1538-4365/abd107", bibcode="2021ApJS..253....7K" + ) + +#Call publications function +# add_publication(db) + +def fix_blobs(db): + with db.engine.begin() as conn: + conn.execute( + db.Sources.update() + .where(db.Sources.c.reference == "Roth") + .values(other_references=None, comments=None) + ) + + +def delete_roth_sources(db): + with db.engine.begin() as conn: + conn.execute(db.Sources.delete().where(db.Sources.c.reference == "Roth")) + + + + +db.inventory("CWISE J000021.45-481314.9", pretty_print=True) + +# WRITE THE JSON FILES +if SAVE_DB: + db.save_database(directory="data/") diff --git a/scripts/ingests/ingest_utils.py b/scripts/ingests/ingest_utils.py index 0174d3588..5ce4ab7eb 100644 --- a/scripts/ingests/ingest_utils.py +++ b/scripts/ingests/ingest_utils.py @@ -5,6 +5,7 @@ from astropy.coordinates import SkyCoord import astropy.units as u from astroquery.gaia import Gaia +from astropy.table import Table from typing import List, Union, Optional import numpy as np import numpy.ma as ma @@ -14,14 +15,35 @@ import dateutil import re import requests +import logging +from sqlalchemy import or_, and_ +import sqlalchemy.exc +from scripts.ingests.utils import ( + SimpleError, + find_source_in_db, + find_publication, + check_internet_connection, +) + +__all__ = [ + "ingest_names", + "ingest_source", + "ingest_sources", + "ingest_spectral_types", + "ingest_parallaxes", + "ingest_proper_motions", + "ingest_photometry", + "ingest_spectra", + "ingest_instrument", + "find_survey_name_in_simbad", +] + +logger = logging.getLogger("SIMPLE") -from scripts.ingests.utils import * - -logger = logging.getLogger('SIMPLE') # NAMES def ingest_names(db, source, other_name): - ''' + """ This function ingests an other name into the Names table Parameters @@ -37,8 +59,8 @@ def ingest_names(db, source, other_name): Returns ------- None - ''' - names_data = [{'source': source, 'other_name': other_name}] + """ + names_data = [{"source": source, "other_name": other_name}] try: with db.engine.connect() as conn: conn.execute(db.Names.insert().values(names_data)) @@ -47,12 +69,23 @@ def ingest_names(db, source, other_name): except sqlalchemy.exc.IntegrityError as e: msg = f"Could not add {names_data} to database. Name is likely a duplicate." logger.warning(msg) - raise SimpleError(msg + '\n' + str(e) + '\n') + raise SimpleError(msg + "\n" + str(e) + "\n") # SOURCES -def ingest_sources(db, sources, references=None, ras=None, decs=None, comments=None, epochs=None, - equinoxes=None, other_references=None, raise_error=True, search_db=True): +def ingest_sources( + db, + sources, + references=None, + ras=None, + decs=None, + comments=None, + epochs=None, + equinoxes=None, + other_references=None, + raise_error=True, + search_db=True, +): """ Script to ingest sources TODO: better support references=None @@ -92,9 +125,9 @@ def ingest_sources(db, sources, references=None, ras=None, decs=None, comments=N # SETUP INPUTS if ras is None and decs is None: - coords = False + coords_provided = False else: - coords = True + coords_provided = True if isinstance(sources, str): n_sources = 1 @@ -102,184 +135,107 @@ def ingest_sources(db, sources, references=None, ras=None, decs=None, comments=N n_sources = len(sources) # Convert single element input values into lists - input_values = [sources, references, ras, decs, epochs, equinoxes, comments, other_references] + input_values = [ + sources, + references, + ras, + decs, + epochs, + equinoxes, + comments, + other_references, + ] for i, input_value in enumerate(input_values): if input_value is None: input_values[i] = [None] * n_sources elif isinstance(input_value, (str, float)): input_values[i] = [input_value] * n_sources - sources, references, ras, decs, epochs, equinoxes, comments, other_references = input_values - - n_added = 0 - n_existing = 0 - n_names = 0 - n_alt_names = 0 - n_skipped = 0 - n_multiples = 0 + ( + sources, + references, + ras, + decs, + epochs, + equinoxes, + comments, + other_references, + ) = input_values + + # TODO: figure out counting + # n_added = 0 + # n_existing = 0 + # n_names = 0 + # n_alt_names = 0 + # n_skipped = 0 + # n_multiples = 0 if n_sources > 1: logger.info(f"Trying to add {n_sources} sources") # Loop over each source and decide to ingest, skip, or add alt name - for i, source in enumerate(sources): - # Find out if source is already in database or not - if coords and search_db: - name_matches = find_source_in_db(db, source, ra=ras[i], dec=decs[i]) - elif search_db: - name_matches = find_source_in_db(db, source) - elif not search_db: - name_matches = [] - else: - name_matches = None - ra = None - dec = None - - if len(name_matches) == 1 and search_db: # Source is already in database - n_existing += 1 - msg1 = f"{i}: Skipping {source}. Already in database as {name_matches[0]}. \n " - logger.debug(msg1) - - # Figure out if ingest name is an alternate name and add - db_matches = db.search_object(source, output_table='Sources', fuzzy_search=False) - if len(db_matches) == 0: - #add other name to names table - ingest_names(db, name_matches[0], source) - n_alt_names += 1 - continue - elif len(name_matches) > 1 and search_db: # Multiple source matches in the database - n_multiples += 1 - msg1 = f"{i} Skipping {source} " - msg = f"{i} More than one match for {source}\n {name_matches}\n" - logger.warning(msg1 + msg) - if raise_error: - raise SimpleError(msg) - else: - continue - elif len(name_matches) == 0 or not search_db: # No match in the database, INGEST! - if coords: # Coordinates were provided as input - ra = ras[i] - dec = decs[i] - epoch = None if ma.is_masked(epochs[i]) else epochs[i] - equinox = None if ma.is_masked(equinoxes[i]) else equinoxes[i] - else: # Try to get coordinates from SIMBAD - simbad_result_table = Simbad.query_object(source) - if simbad_result_table is None: - n_skipped += 1 - ra = None - dec = None - msg = f"{i}: Skipping: {source}. Coordinates are needed and could not be retrieved from SIMBAD. \n" - logger.warning(msg) - if raise_error: - raise SimpleError(msg) - else: - continue - elif len(simbad_result_table) == 1: - simbad_coords = simbad_result_table['RA'][0] + ' ' + simbad_result_table['DEC'][0] - simbad_skycoord = SkyCoord(simbad_coords, unit=(u.hourangle, u.deg)) - ra = simbad_skycoord.to_string(style='decimal').split()[0] - dec = simbad_skycoord.to_string(style='decimal').split()[1] - epoch = '2000' # Default coordinates from SIMBAD are epoch 2000. - equinox = 'J2000' # Default frame from SIMBAD is IRCS and J2000. - msg = f"Coordinates retrieved from SIMBAD {ra}, {dec}" - logger.debug(msg) - else: - n_skipped += 1 - ra = None - dec = None - msg = f"{i}: Skipping: {source}. Coordinates are needed and could not be retrieved from SIMBAD. \n" - logger.warning(msg) - if raise_error: - raise SimpleError(msg) - else: - continue - - logger.debug(f"{i}: Ingesting {source}. Not already in database. ") + for source_counter, source in enumerate(sources): + + logger.debug(f"{source_counter}: Trying to ingest {source}") + + reference = references[source_counter] + other_reference = other_references[source_counter] + comment = ( + None if ma.is_masked(comments[source_counter]) else comments[source_counter] + ) + + if coords_provided: + ra = ras[source_counter] + dec = decs[source_counter] + epoch = ( + None if ma.is_masked(epochs[source_counter]) else epochs[source_counter] + ) + equinox = ( + None + if ma.is_masked(equinoxes[source_counter]) + else equinoxes[source_counter] + ) + + ingest_source( + db, + source, + reference=reference, + ra=ra, + dec=dec, + epoch=epoch, + equinox=equinox, + other_reference=other_reference, + comment=comment, + raise_error=raise_error, + search_db=search_db, + ) else: - msg = f"{i}: unexpected condition encountered ingesting {source}" - logger.error(msg) - raise SimpleError(msg) - - # Construct data to be added - source_data = [{'source': source, - 'ra': ra, - 'dec': dec, - 'reference': references[i], - 'epoch': epoch, - 'equinox': equinox, - 'other_references': other_references[i], - 'comments': None if ma.is_masked(comments[i]) else comments[i]}] - names_data = [{'source': source, - 'other_name': source}] - - # Try to add the source to the database - try: - with db.engine.connect() as conn: - conn.execute(db.Sources.insert().values(source_data)) - conn.commit() - n_added += 1 - msg = f"Added {str(source_data)}" - logger.debug(msg) - except sqlalchemy.exc.IntegrityError: - if ma.is_masked(source_data[0]['reference']): # check if reference is blank - msg = f"{i}: Skipping: {source}. Discovery reference is blank. \n" - msg2 = f"\n {str(source_data)}\n" - logger.warning(msg) - logger.debug(msg2) - n_skipped += 1 - if raise_error: - raise SimpleError(msg + msg2) - else: - continue - elif db.query(db.Publications).filter(db.Publications.c.publication == references[i]).count() == 0: - # check if reference is in Publications table - msg = f"{i}: Skipping: {source}. Discovery reference {references[i]} is not in Publications table. \n" \ - f"(Add it with add_publication function.) \n " - msg2 = f"\n {str(source_data)}\n" - logger.warning(msg) - logger.debug(msg2) - n_skipped += 1 - if raise_error: - raise SimpleError(msg + msg2) - else: - continue - else: - msg = f"{i}: Skipping: {source}. Not sure why." - msg2 = f"\n {str(source_data)} " - logger.warning(msg) - logger.debug(msg2) - n_skipped += 1 - if raise_error: - raise SimpleError(msg + msg2) - else: - continue - - # Try to add the source name to the Names table - try: - ingest_names(db, source, source) - n_names += 1 - except sqlalchemy.exc.IntegrityError: - msg = f"{i}: Could not add {names_data} to database" - logger.warning(msg) - if raise_error: - raise SimpleError(msg) - else: - continue - - if n_sources > 1: - logger.info(f"Sources added to database: {n_added}") - logger.info(f"Names added to database: {n_names} \n") - logger.info(f"Sources already in database: {n_existing}") - logger.info(f"Alt Names added to database: {n_alt_names}") - logger.info(f"Sources NOT added to database because multiple matches: {n_multiples}") - logger.info(f"Sources NOT added to database: {n_skipped} \n") - - if n_added != n_names: - msg = f"Number added should equal names added." - raise SimpleError(msg) - - if n_added + n_existing + n_multiples + n_skipped != n_sources: - msg = f"Number added + Number skipped doesn't add up to total sources" - raise SimpleError(msg) + ingest_source( + db, + source, + reference=reference, + other_reference=other_reference, + comment=comment, + raise_error=raise_error, + search_db=search_db, + ) + + # if n_sources > 1: + # logger.info(f"Sources added to database: {n_added}") + # logger.info(f"Names added to database: {n_names} \n") + # logger.info(f"Sources already in database: {n_existing}") + # logger.info(f"Alt Names added to database: {n_alt_names}") + # logger.info( + # f"Sources NOT added to database because multiple matches: {n_multiples}" + # ) + # logger.info(f"Sources NOT added to database: {n_skipped} \n") + + # if n_added != n_names: + # msg = f"Number added should equal names added." + # raise SimpleError(msg) + + # if n_added + n_existing + n_multiples + n_skipped != n_sources: + # msg = f"Number added + Number skipped doesn't add up to total sources" + # raise SimpleError(msg) return @@ -307,54 +263,66 @@ def find_survey_name_in_simbad(sources, desig_prefix, source_id_index=None): n_sources = len(sources) Simbad.reset_votable_fields() - Simbad.add_votable_fields('typed_id') # keep search term in result table - Simbad.add_votable_fields('ids') # add all SIMBAD identifiers as an output column + Simbad.add_votable_fields("typed_id") # keep search term in result table + Simbad.add_votable_fields("ids") # add all SIMBAD identifiers as an output column logger.info("simbad query started") - result_table = Simbad.query_objects(sources['source']) + result_table = Simbad.query_objects(sources["source"]) logger.info("simbad query ended") - ind = result_table['SCRIPT_NUMBER_ID'] > 0 # find indexes which contain results - simbad_ids = result_table['TYPED_ID', 'IDS'][ind] + ind = result_table["SCRIPT_NUMBER_ID"] > 0 # find indexes which contain results + simbad_ids = result_table["TYPED_ID", "IDS"][ind] db_names = [] simbad_designations = [] source_ids = [] for row in simbad_ids: - db_name = row['TYPED_ID'] - ids = row['IDS'].split('|') + db_name = row["TYPED_ID"] + ids = row["IDS"].split("|") designation = [i for i in ids if desig_prefix in i] if designation: - logger.debug(f'{db_name}, {designation[0]}') + logger.debug(f"{db_name}, {designation[0]}") db_names.append(db_name) if len(designation) == 1: simbad_designations.append(designation[0]) else: simbad_designations.append(designation[0]) - logger.warning(f'more than one designation matched, {designation}') + logger.warning(f"more than one designation matched, {designation}") if source_id_index is not None: source_id = designation[0].split()[source_id_index] source_ids.append(int(source_id)) # convert to int since long in Gaia n_matches = len(db_names) - logger.info(f"Found, {n_matches}, {desig_prefix}, sources for, {n_sources}, sources") + logger.info( + f"Found, {n_matches}, {desig_prefix}, sources for, {n_sources}, sources" + ) if source_id_index is not None: - result_table = Table([db_names, simbad_designations, source_ids], - names=('db_names', 'designation', 'source_id')) + result_table = Table( + [db_names, simbad_designations, source_ids], + names=("db_names", "designation", "source_id"), + ) else: - result_table = Table([db_names, simbad_designations], - names=('db_names', 'designation')) + result_table = Table( + [db_names, simbad_designations], names=("db_names", "designation") + ) return result_table # SPECTRAL TYPES -def ingest_spectral_types(db, sources, spectral_types, references, regimes, spectral_type_error=None, - comments=None): +def ingest_spectral_types( + db, + sources, + spectral_types, + references, + regimes, + spectral_type_error=None, + comments=None, +): """ Script to ingest spectral types Parameters @@ -383,14 +351,28 @@ def ingest_spectral_types(db, sources, spectral_types, references, regimes, spec n_sources = len(sources) # Convert single element input value to list - input_values = [sources, spectral_types, spectral_type_error, regimes, comments, references] + input_values = [ + sources, + spectral_types, + spectral_type_error, + regimes, + comments, + references, + ] for i, input_value in enumerate(input_values): if input_value is None: input_values[i] = [None] * n_sources elif isinstance(input_value, str): input_values[i] = [input_value] * n_sources # Convert single element input value to list - sources, spectral_types, spectral_type_error, regimes, comments, references = input_values + ( + sources, + spectral_types, + spectral_type_error, + regimes, + comments, + references, + ) = input_values n_added = 0 n_skipped = 0 @@ -402,21 +384,27 @@ def ingest_spectral_types(db, sources, spectral_types, references, regimes, spec # Spectral Type data is in the database if len(db_name) != 1: - msg = f"No unique source match for {source} in the database " \ - f"(with SpT: {spectral_types[i]} from {references[i]})" + msg = ( + f"No unique source match for {source} in the database " + f"(with SpT: {spectral_types[i]} from {references[i]})" + ) raise SimpleError(msg) else: db_name = db_name[0] adopted = None - source_spt_data = db.query(db.SpectralTypes).filter(db.SpectralTypes.c.source == db_name).table() + source_spt_data = ( + db.query(db.SpectralTypes) + .filter(db.SpectralTypes.c.source == db_name) + .table() + ) if source_spt_data is None or len(source_spt_data) == 0: adopted: True logger.debug("No Spectral Type data for this source in the database") elif len(source_spt_data) > 0: # Spectral Type Data already exists - dupe_ind = source_spt_data['reference'] == references[i] + dupe_ind = source_spt_data["reference"] == references[i] if sum(dupe_ind): logger.debug(f"Duplicate measurement\n, {source_spt_data[dupe_ind]}") else: @@ -424,26 +412,40 @@ def ingest_spectral_types(db, sources, spectral_types, references, regimes, spec if logger.level == 10: source_spt_data.pprint_all() - adopted_ind = source_spt_data['adopted'] == 1 + adopted_ind = source_spt_data["adopted"] == 1 if sum(adopted_ind): old_adopted = source_spt_data[adopted_ind] - if spectral_type_error[i] < min(source_spt_data['spectral_type_error']): + if spectral_type_error[i] < min(source_spt_data["spectral_type_error"]): adopted = True if old_adopted: with db.engine.connect() as conn: conn.execute( - db.SpectralTypes. \ - update(). \ - where(and_(db.SpectralTypes.c.source == old_adopted['source'][0], - db.SpectralTypes.c.reference == old_adopted['reference'][0])). \ - values(adopted=False) + db.SpectralTypes.update() + .where( + and_( + db.SpectralTypes.c.source + == old_adopted["source"][0], + db.SpectralTypes.c.reference + == old_adopted["reference"][0], + ) ) + .values(adopted=False) + ) conn.commit() # check that adopted flag is successfully changed - old_adopted_data = db.query(db.SpectralTypes).filter( - and_(db.SpectralTypes.c.source == old_adopted['source'][0], - db.SpectralTypes.c.reference == old_adopted['reference'][0])).table() + old_adopted_data = ( + db.query(db.SpectralTypes) + .filter( + and_( + db.SpectralTypes.c.source + == old_adopted["source"][0], + db.SpectralTypes.c.reference + == old_adopted["reference"][0], + ) + ) + .table() + ) logger.debug("Old adopted measurement unset") if logger.level == 10: old_adopted_data.pprint_all() @@ -460,23 +462,37 @@ def ingest_spectral_types(db, sources, spectral_types, references, regimes, spec logger.debug(msg) # Construct the data to be added - spt_data = [{'source': db_name, - 'spectral_type_string': spectral_types[i], - 'spectral_type_code': spectral_type_code, - 'spectral_type_error': spectral_type_error[i], - 'regime': regimes[i], - 'adopted': adopted, - 'comments': comments[i], - 'reference': references[i]}] + spt_data = [ + { + "source": db_name, + "spectral_type_string": spectral_types[i], + "spectral_type_code": spectral_type_code, + "spectral_type_error": spectral_type_error[i], + "regime": regimes[i], + "adopted": adopted, + "comments": comments[i], + "reference": references[i], + } + ] # Check if the entry already exists; if so: skip adding it - check = db.query(db.SpectralTypes.c.source).filter(and_(db.SpectralTypes.c.source == db_name, - db.SpectralTypes.c.regime == regimes[i], - db.SpectralTypes.c.reference == references[i])).count() + check = ( + db.query(db.SpectralTypes.c.source) + .filter( + and_( + db.SpectralTypes.c.source == db_name, + db.SpectralTypes.c.regime == regimes[i], + db.SpectralTypes.c.reference == references[i], + ) + ) + .count() + ) if check == 1: n_skipped += 1 - logger.info(f'Spectral type for {db_name} already in the database: skipping insert ' - f'{spt_data}') + logger.info( + f"Spectral type for {db_name} already in the database: skipping insert " + f"{spt_data}" + ) continue logger.debug(f"Trying to insert {spt_data} into Spectral Types table ") @@ -488,8 +504,13 @@ def ingest_spectral_types(db, sources, spectral_types, references, regimes, spec msg = f"Added {str(spt_data)}" logger.debug(msg) except sqlalchemy.exc.IntegrityError as e: - if db.query(db.Publications).filter(db.Publications.c.reference == references[i]).count() == 0: - msg = f"The publication does not exist in the database" + if ( + db.query(db.Publications) + .filter(db.Publications.c.reference == references[i]) + .count() + == 0 + ): + msg = f"The publication {references[i]} does not exist in the database" msg1 = f"Add it with ingest_publication function." logger.debug(msg + msg1) raise SimpleError(msg) @@ -502,8 +523,7 @@ def ingest_spectral_types(db, sources, spectral_types, references, regimes, spec logger.error(msg) raise SimpleError(msg) - msg = f"Spectral types added: {n_added} \n" \ - f"Spectral Types skipped: {n_skipped}" + msg = f"Spectral types added: {n_added} \n" f"Spectral Types skipped: {n_skipped}" logger.info(msg) @@ -533,25 +553,25 @@ def convert_spt_string_to_code(spectral_types): continue # identify main spectral class, loop over any prefix text to identify MLTY for i, item in enumerate(spt): - if item == 'M': + if item == "M": spt_code = 60 break - elif item == 'L': + elif item == "L": spt_code = 70 break - elif item == 'T': + elif item == "T": spt_code = 80 break - elif item == 'Y': + elif item == "Y": spt_code = 90 break else: # only trigger if not MLTY i = 0 # find integer or decimal subclass and add to spt_code - if re.search('\d*\.?\d+', spt[i+1:]) is None: + if re.search("\d*\.?\d+", spt[i + 1 :]) is None: spt_code = spt_code else: - spt_code += float(re.findall('\d*\.?\d+', spt[i + 1:])[0]) + spt_code += float(re.findall("\d*\.?\d+", spt[i + 1 :])[0]) spectral_type_codes.append(spt_code) return spectral_type_codes @@ -576,21 +596,21 @@ def convert_spt_code_to_string_to_code(spectral_codes, decimals=1): spectral_types = [] for spt in spectral_codes: - spt_type = '' + spt_type = "" # Identify major type if 60 <= spt < 70: - spt_type = 'M' + spt_type = "M" elif 70 <= spt < 80: - spt_type = 'L' + spt_type = "L" elif 80 <= spt < 90: - spt_type = 'T' + spt_type = "T" elif 90 <= spt < 100: - spt_type = 'Y' + spt_type = "Y" # Numeric part of type - format = f'.{decimals}f' - spt_type = f'{spt_type}{spt % 10:{format}}' + format = f".{decimals}f" + spt_type = f"{spt_type}{spt % 10:{format}}" logger.debug(f"Converting: {spt} -> {spt_type}") spectral_types.append(spt_type) @@ -646,7 +666,8 @@ def ingest_parallaxes(db, sources, plxs, plx_errs, plx_refs, comments=None): n_added = 0 - for i, source in enumerate(sources): # loop through sources with parallax data to ingest + # loop through sources with parallax data to ingest + for i, source in enumerate(sources): db_name = find_source_in_db(db, source) if len(db_name) != 1: @@ -658,16 +679,19 @@ def ingest_parallaxes(db, sources, plxs, plx_errs, plx_refs, comments=None): # Search for existing parallax data and determine if this is the best # If no previous measurement exists, set the new one to the Adopted measurement adopted = None - source_plx_data: Table = db.query(db.Parallaxes).filter(db.Parallaxes.c.source == db_name).table() + source_plx_data: Table = ( + db.query(db.Parallaxes).filter(db.Parallaxes.c.source == db_name).table() + ) if source_plx_data is None or len(source_plx_data) == 0: - # if there's no other measurements in the database, set new data Adopted = True + # if there's no other measurements in the database, + # set new data Adopted = True adopted = True # old_adopted = None # not used logger.debug("No other measurement") elif len(source_plx_data) > 0: # Parallax data already exists # check for duplicate measurement - dupe_ind = source_plx_data['reference'] == plx_refs[i] + dupe_ind = source_plx_data["reference"] == plx_refs[i] if sum(dupe_ind): logger.debug(f"Duplicate measurement\n, {source_plx_data[dupe_ind]}") continue @@ -677,28 +701,41 @@ def ingest_parallaxes(db, sources, plxs, plx_errs, plx_refs, comments=None): source_plx_data.pprint_all() # check for previous adopted measurement and find new adopted - adopted_ind = source_plx_data['adopted'] == 1 + adopted_ind = source_plx_data["adopted"] == 1 if sum(adopted_ind): old_adopted = source_plx_data[adopted_ind] # if errors of new data are less than other measurements, set Adopted = True. - if plx_errs[i] < min(source_plx_data['parallax_error']): + if plx_errs[i] < min(source_plx_data["parallax_error"]): adopted = True # unset old adopted if old_adopted: with db.engine.connect() as conn: conn.execute( - db.Parallaxes. \ - update(). \ - where(and_(db.Parallaxes.c.source == old_adopted['source'][0], - db.Parallaxes.c.reference == old_adopted['reference'][0])). \ - values(adopted=False) + db.Parallaxes.update() + .where( + and_( + db.Parallaxes.c.source + == old_adopted["source"][0], + db.Parallaxes.c.reference + == old_adopted["reference"][0], + ) ) + .values(adopted=False) + ) conn.commit() # check that adopted flag is successfully changed - old_adopted_data = db.query(db.Parallaxes).filter( - and_(db.Parallaxes.c.source == old_adopted['source'][0], - db.Parallaxes.c.reference == old_adopted['reference'][0])).table() + old_adopted_data = ( + db.query(db.Parallaxes) + .filter( + and_( + db.Parallaxes.c.source == old_adopted["source"][0], + db.Parallaxes.c.reference + == old_adopted["reference"][0], + ) + ) + .table() + ) logger.debug("Old adopted measurement unset") if logger.level == 10: old_adopted_data.pprint_all() @@ -706,17 +743,21 @@ def ingest_parallaxes(db, sources, plxs, plx_errs, plx_refs, comments=None): adopted = False logger.debug(f"The new measurement's adopted flag is:, {adopted}") else: - msg = 'Unexpected state' + msg = "Unexpected state" logger.error(msg) raise RuntimeError(msg) # Construct data to be added - parallax_data = [{'source': db_name, - 'parallax': plxs[i], - 'parallax_error': plx_errs[i], - 'reference': plx_refs[i], - 'adopted': adopted, - 'comments': comments[i]}] + parallax_data = [ + { + "source": db_name, + "parallax": plxs[i], + "parallax_error": plx_errs[i], + "reference": plx_refs[i], + "adopted": adopted, + "comments": comments[i], + } + ] logger.debug(f"{parallax_data}") @@ -725,13 +766,14 @@ def ingest_parallaxes(db, sources, plxs, plx_errs, plx_refs, comments=None): conn.execute(db.Parallaxes.insert().values(parallax_data)) conn.commit() n_added += 1 - logger.info(f"Parallax added to database: \n " - f"{parallax_data}") + logger.info(f"Parallax added to database: \n " f"{parallax_data}") except sqlalchemy.exc.IntegrityError: - msg = "The source may not exist in Sources table.\n" \ - "The parallax reference may not exist in Publications table. " \ - "Add it with add_publication function. \n" \ - "The parallax measurement may be a duplicate." + msg = ( + "The source may not exist in Sources table.\n" + "The parallax reference may not exist in Publications table. " + "Add it with add_publication function. \n" + "The parallax measurement may be a duplicate." + ) logger.error(msg) raise SimpleError(msg) @@ -741,7 +783,9 @@ def ingest_parallaxes(db, sources, plxs, plx_errs, plx_refs, comments=None): # PROPER MOTIONS -def ingest_proper_motions(db, sources, pm_ras, pm_ra_errs, pm_decs, pm_dec_errs, pm_references): +def ingest_proper_motions( + db, sources, pm_ras, pm_ra_errs, pm_decs, pm_dec_errs, pm_references +): """ Parameters @@ -784,7 +828,6 @@ def ingest_proper_motions(db, sources, pm_ras, pm_ra_errs, pm_decs, pm_dec_errs, n_added = 0 for i, source in enumerate(sources): - db_name = find_source_in_db(db, source) if len(db_name) != 1: @@ -796,52 +839,73 @@ def ingest_proper_motions(db, sources, pm_ras, pm_ra_errs, pm_decs, pm_dec_errs, # Search for existing proper motion data and determine if this is the best # If no previous measurement exists, set the new one to the Adopted measurement # adopted = None # not used - source_pm_data = db.query(db.ProperMotions).filter(db.ProperMotions.c.source == db_name).table() + source_pm_data = ( + db.query(db.ProperMotions) + .filter(db.ProperMotions.c.source == db_name) + .table() + ) if source_pm_data is None or len(source_pm_data) == 0: - # if there's no other measurements in the database, set new data Adopted = True + # if there's no other measurements in the database, + # set new data Adopted = True adopted = True elif len(source_pm_data) > 0: - # check to see if other measurement is a duplicate of the new data - dupe_ind = source_pm_data['reference'] == pm_references[i] + dupe_ind = source_pm_data["reference"] == pm_references[i] if sum(dupe_ind): logger.debug(f"Duplicate measurement\n, {source_pm_data}") continue # check for previous adopted measurement - adopted_ind = source_pm_data['adopted'] == 1 + adopted_ind = source_pm_data["adopted"] == 1 if sum(adopted_ind): old_adopted = source_pm_data[adopted_ind] else: old_adopted = None # if errors of new data are less than other measurements, set Adopted = True. - if pm_ra_errs[i] < min(source_pm_data['mu_ra_error']) and pm_dec_errs[i] < min( - source_pm_data['mu_dec_error']): + if pm_ra_errs[i] < min(source_pm_data["mu_ra_error"]) and pm_dec_errs[ + i + ] < min(source_pm_data["mu_dec_error"]): adopted = True # unset old adopted if it exists if old_adopted: with db.engine.connect() as conn: conn.execute( - db.ProperMotions. \ - update(). \ - where(and_(db.ProperMotions.c.source == old_adopted['source'][0], - db.ProperMotions.c.reference == old_adopted['reference'][0])). \ - values(adopted=False) + db.ProperMotions.update() + .where( + and_( + db.ProperMotions.c.source + == old_adopted["source"][0], + db.ProperMotions.c.reference + == old_adopted["reference"][0], + ) ) + .values(adopted=False) + ) conn.commit() else: adopted = False - # if no previous adopted measurement, set adopted to the measurement with the smallest errors - if not adopted and not old_adopted and \ - min(source_pm_data['mu_ra_error']) < pm_ra_errs[i] and \ - min(source_pm_data['mu_dec_error']) < pm_dec_errs[i]: - adopted_pm = db.ProperMotions.update().where(and_(db.ProperMotions.c.source == db_name, - db.ProperMotions.c.mu_ra_error == min( - source_pm_data['mu_ra_error']), - db.ProperMotions.c.mu_dec_error == min( - source_pm_data['mu_dec_error']))). \ - values(adopted=True) + # if no previous adopted measurement, + # set adopted to the measurement with the smallest errors + if ( + not adopted + and not old_adopted + and min(source_pm_data["mu_ra_error"]) < pm_ra_errs[i] + and min(source_pm_data["mu_dec_error"]) < pm_dec_errs[i] + ): + adopted_pm = ( + db.ProperMotions.update() + .where( + and_( + db.ProperMotions.c.source == db_name, + db.ProperMotions.c.mu_ra_error + == min(source_pm_data["mu_ra_error"]), + db.ProperMotions.c.mu_dec_error + == min(source_pm_data["mu_dec_error"]), + ) + ) + .values(adopted=True) + ) with db.engine.connect() as conn: conn.execute(adopted_pm) conn.commit() @@ -850,19 +914,23 @@ def ingest_proper_motions(db, sources, pm_ras, pm_ra_errs, pm_decs, pm_dec_errs, source_pm_data.pprint_all() else: - msg = 'Unexpected state' + msg = "Unexpected state" logger.error(msg) raise RuntimeError(msg) # Construct data to be added - pm_data = [{'source': db_name, - 'mu_ra': pm_ras[i], - 'mu_ra_error': pm_ra_errs[i], - 'mu_dec': pm_decs[i], - 'mu_dec_error': pm_dec_errs[i], - 'adopted': adopted, - 'reference': pm_references[i]}] - logger.debug(f'Proper motion data to add: {pm_data}') + pm_data = [ + { + "source": db_name, + "mu_ra": pm_ras[i], + "mu_ra_error": pm_ra_errs[i], + "mu_dec": pm_decs[i], + "mu_dec_error": pm_dec_errs[i], + "adopted": adopted, + "reference": pm_references[i], + } + ] + logger.debug(f"Proper motion data to add: {pm_data}") try: with db.engine.connect() as conn: @@ -870,15 +938,21 @@ def ingest_proper_motions(db, sources, pm_ras, pm_ra_errs, pm_decs, pm_dec_errs, conn.commit() n_added += 1 except sqlalchemy.exc.IntegrityError: - msg = "The source may not exist in Sources table.\n" \ - "The proper motion reference may not exist in Publications table. " \ - "Add it with add_publication function. \n" \ - "The proper motion measurement may be a duplicate." + msg = ( + "The source may not exist in Sources table.\n" + "The proper motion reference may not exist in Publications table. " + "Add it with add_publication function. \n" + "The proper motion measurement may be a duplicate." + ) logger.error(msg) raise SimpleError(msg) - updated_source_pm_data = db.query(db.ProperMotions).filter(db.ProperMotions.c.source == db_name).table() - logger.info('Updated proper motion data:') + updated_source_pm_data = ( + db.query(db.ProperMotions) + .filter(db.ProperMotions.c.source == db_name) + .table() + ) + logger.info("Updated proper motion data:") if logger.level == 20: # Info = 20, Debug = 10 updated_source_pm_data.pprint_all() @@ -886,8 +960,18 @@ def ingest_proper_motions(db, sources, pm_ras, pm_ra_errs, pm_decs, pm_dec_errs, # PHOTOMETRY -def ingest_photometry(db, sources, bands, magnitudes, magnitude_errors, reference, - telescope=None, epoch=None, comments=None, raise_error=True): +def ingest_photometry( + db, + sources, + bands, + magnitudes, + magnitude_errors, + reference, + telescope=None, + epoch=None, + comments=None, + raise_error=True, +): """ TODO: Write Docstring @@ -936,13 +1020,19 @@ def ingest_photometry(db, sources, bands, magnitudes, magnitude_errors, referenc magnitudes, magnitude_errors = input_float_values if n_sources != len(magnitudes) or n_sources != len(magnitude_errors): - msg = f"N Sources: {len(sources)}, " \ - f"N Magnitudes: {len(magnitudes)}, N Mag errors: {len(magnitude_errors)}," \ - f"\nSources, magnitudes, and magnitude error lists should all be same length" + msg = ( + f"N Sources: {len(sources)}, " + f"N Magnitudes: {len(magnitudes)}, N Mag errors: {len(magnitude_errors)}," + f"\nSources, magnitudes, and magnitude error lists should all be same length" + ) logger.error(msg) raise RuntimeError(msg) - if n_sources != len(reference) or n_sources != len(telescope) or n_sources != len(bands): + if ( + n_sources != len(reference) + or n_sources != len(telescope) + or n_sources != len(bands) + ): msg = "All lists should be same length" logger.error(msg) raise RuntimeError(msg) @@ -965,25 +1055,30 @@ def ingest_photometry(db, sources, bands, magnitudes, magnitude_errors, referenc mag_error = str(magnitude_errors[i]) # Construct data to be added - photometry_data = [{'source': db_name, - 'band': bands[i], - 'magnitude': str(magnitudes[i]), # Convert to string to maintain significant digits - 'magnitude_error': mag_error, - 'telescope': telescope[i], - 'epoch': epoch, - 'comments': comments, - 'reference': reference[i]}] - logger.debug(f'Photometry data: {photometry_data}') + photometry_data = [ + { + "source": db_name, + "band": bands[i], + "magnitude": str( + magnitudes[i] + ), # Convert to string to maintain significant digits + "magnitude_error": mag_error, + "telescope": telescope[i], + "epoch": epoch, + "comments": comments, + "reference": reference[i], + } + ] + logger.debug(f"Photometry data: {photometry_data}") try: with db.engine.connect() as conn: conn.execute(db.Photometry.insert().values(photometry_data)) conn.commit() n_added += 1 - logger.info(f"Photometry measurement added: \n" - f"{photometry_data}") + logger.info(f"Photometry measurement added: \n" f"{photometry_data}") except sqlalchemy.exc.IntegrityError as e: - if 'UNIQUE constraint failed:' in str(e): + if "UNIQUE constraint failed:" in str(e): msg = "The measurement may be a duplicate." if raise_error: logger.error(msg) @@ -992,9 +1087,11 @@ def ingest_photometry(db, sources, bands, magnitudes, magnitude_errors, referenc logger.warning(msg) continue else: - msg = "The source may not exist in Sources table.\n" \ - "The reference may not exist in the Publications table. " \ - "Add it with add_publication function." + msg = ( + "The source may not exist in Sources table.\n" + "The reference may not exist in the Publications table. " + "Add it with add_publication function." + ) logger.error(msg) raise SimpleError(msg) @@ -1004,9 +1101,24 @@ def ingest_photometry(db, sources, bands, magnitudes, magnitude_errors, referenc # SPECTRA -def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes, obs_dates, references,original_spectra=None, - wavelength_units=None, flux_units=None, wavelength_order=None, - comments=None, other_references=None, raise_error=True): +def ingest_spectra( + db, + sources, + spectra, + regimes, + telescopes, + instruments, + modes, + obs_dates, + references, + original_spectra=None, + wavelength_units=None, + flux_units=None, + wavelength_order=None, + comments=None, + other_references=None, + raise_error=True, +): """ Parameters @@ -1050,15 +1162,37 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes if isinstance(spectra, str): spectra = [spectra] - input_values = [regimes, telescopes, instruments, modes, obs_dates, wavelength_order, wavelength_units, flux_units, - references,comments, other_references] + input_values = [ + regimes, + telescopes, + instruments, + modes, + obs_dates, + wavelength_order, + wavelength_units, + flux_units, + references, + comments, + other_references, + ] for i, input_value in enumerate(input_values): if isinstance(input_value, str): input_values[i] = [input_value] * len(sources) elif isinstance(input_value, type(None)): input_values[i] = [None] * len(sources) - regimes, telescopes, instruments, modes, obs_dates, wavelength_order, wavelength_units, flux_units, \ - references, comments, other_references = input_values + ( + regimes, + telescopes, + instruments, + modes, + obs_dates, + wavelength_order, + wavelength_units, + flux_units, + references, + comments, + other_references, + ) = input_values n_spectra = len(spectra) n_skipped = 0 @@ -1067,7 +1201,7 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes n_added = 0 n_blank = 0 - msg = f'Trying to add {n_spectra} spectra' + msg = f"Trying to add {n_spectra} spectra" logger.info(msg) for i, source in enumerate(sources): @@ -1087,12 +1221,16 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes internet = check_internet_connection() if internet: request_response = requests.head(spectra[i]) - status_code = request_response.status_code # The website is up if the status code is 200 + status_code = ( + request_response.status_code + ) # The website is up if the status code is 200 if status_code != 200: n_skipped += 1 - msg = "The spectrum location does not appear to be valid: \n" \ - f'spectrum: {spectra[i]} \n' \ - f'status code: {status_code}' + msg = ( + "The spectrum location does not appear to be valid: \n" + f"spectrum: {spectra[i]} \n" + f"status code: {status_code}" + ) logger.error(msg) if raise_error: raise SimpleError(msg) @@ -1106,9 +1244,11 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes status_code1 = request_response1.status_code if status_code1 != 200: n_skipped += 1 - msg = "The spectrum location does not appear to be valid: \n" \ - f'spectrum: {original_spectra[i]} \n' \ - f'status code: {status_code1}' + msg = ( + "The spectrum location does not appear to be valid: \n" + f"spectrum: {original_spectra[i]} \n" + f"status code: {status_code1}" + ) logger.error(msg) if raise_error: raise SimpleError(msg) @@ -1122,13 +1262,17 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes raise SimpleError(msg) # Find what spectra already exists in database for this source - source_spec_data = db.query(db.Spectra).filter(db.Spectra.c.source == db_name).table() + source_spec_data = ( + db.query(db.Spectra).filter(db.Spectra.c.source == db_name).table() + ) # SKIP if observation date is blank # TODO: try to populate obs date from meta data in spectrum file - if ma.is_masked(obs_dates[i]) or obs_dates[i] == '': + if ma.is_masked(obs_dates[i]) or obs_dates[i] == "": obs_date = None - missing_obs_msg = f"Skipping spectrum with missing observation date: {source} \n" + missing_obs_msg = ( + f"Skipping spectrum with missing observation date: {source} \n" + ) missing_row_spe = f"{source, obs_dates[i], references[i]} \n" logger.info(missing_obs_msg) logger.debug(missing_row_spe) @@ -1136,7 +1280,9 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes continue else: try: - obs_date = pd.to_datetime(obs_dates[i]) # TODO: Another method that doesn't require pandas? + obs_date = pd.to_datetime( + obs_dates[i] + ) # TODO: Another method that doesn't require pandas? except ValueError: n_skipped += 1 if raise_error: @@ -1155,22 +1301,32 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes continue # TODO: make it possible to ingest units and order - row_data = [{'source': db_name, - 'spectrum': spectra[i], - 'original_spectrum': None, # if ma.is_masked(original_spectra[i]) or isinstance(original_spectra,None) - # else original_spectra[i], - 'local_spectrum': None, # if ma.is_masked(local_spectra[i]) else local_spectra[i], - 'regime': regimes[i], - 'telescope': telescopes[i], - 'instrument': None if ma.is_masked(instruments[i]) else instruments[i], - 'mode': None if ma.is_masked(modes[i]) else modes[i], - 'observation_date': obs_date, - 'wavelength_units': None if ma.is_masked(wavelength_units[i]) else wavelength_units[i], - 'flux_units': None if ma.is_masked(flux_units[i]) else flux_units[i], - 'wavelength_order': None if ma.is_masked(wavelength_order[i]) else wavelength_order[i], - 'comments': None if ma.is_masked(comments[i]) else comments[i], - 'reference': references[i], - 'other_references': None if ma.is_masked(other_references[i]) else other_references[i]}] + row_data = [ + { + "source": db_name, + "spectrum": spectra[i], + "original_spectrum": None, # if ma.is_masked(original_spectra[i]) or isinstance(original_spectra,None) + # else original_spectra[i], + "local_spectrum": None, # if ma.is_masked(local_spectra[i]) else local_spectra[i], + "regime": regimes[i], + "telescope": telescopes[i], + "instrument": None if ma.is_masked(instruments[i]) else instruments[i], + "mode": None if ma.is_masked(modes[i]) else modes[i], + "observation_date": obs_date, + "wavelength_units": None + if ma.is_masked(wavelength_units[i]) + else wavelength_units[i], + "flux_units": None if ma.is_masked(flux_units[i]) else flux_units[i], + "wavelength_order": None + if ma.is_masked(wavelength_order[i]) + else wavelength_order[i], + "comments": None if ma.is_masked(comments[i]) else comments[i], + "reference": references[i], + "other_references": None + if ma.is_masked(other_references[i]) + else other_references[i], + } + ] logger.debug(row_data) try: @@ -1179,7 +1335,6 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes conn.commit() n_added += 1 except sqlalchemy.exc.IntegrityError as e: - if "CHECK constraint failed: regime" in str(e): msg = f"Regime provided is not in schema: {regimes[i]}" logger.error(msg) @@ -1187,26 +1342,50 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes raise SimpleError(msg) else: continue - if db.query(db.Publications).filter(db.Publications.c.publication == references[i]).count() == 0: - msg = f"Spectrum for {source} could not be added to the database because the reference {references[i]} is not in Publications table. \n" \ - f"(Add it with ingest_publication function.) \n " + if ( + db.query(db.Publications) + .filter(db.Publications.c.publication == references[i]) + .count() + == 0 + ): + msg = ( + f"Spectrum for {source} could not be added to the database because the reference {references[i]} is not in Publications table. \n" + f"(Add it with ingest_publication function.) \n " + ) logger.warning(msg) if raise_error: raise SimpleError(msg) else: continue # check telescope, instrument, mode exists - telescope = db.query(db.Telescopes).filter(db.Telescopes.c.name == row_data[0]['telescope']).table() - instrument = db.query(db.Instruments).filter(db.Instruments.c.name == row_data[0]['instrument']).table() - mode = db.query(db.Modes).filter(db.Modes.c.name == row_data[0]['mode']).table() + telescope = ( + db.query(db.Telescopes) + .filter(db.Telescopes.c.name == row_data[0]["telescope"]) + .table() + ) + instrument = ( + db.query(db.Instruments) + .filter(db.Instruments.c.name == row_data[0]["instrument"]) + .table() + ) + mode = ( + db.query(db.Modes) + .filter(db.Modes.c.name == row_data[0]["mode"]) + .table() + ) if len(source_spec_data) > 0: # Spectra data already exists # check for duplicate measurement - ref_dupe_ind = source_spec_data['reference'] == references[i] - date_dupe_ind = source_spec_data['observation_date'] == obs_date - instrument_dupe_ind = source_spec_data['instrument'] == instruments[i] - mode_dupe_ind = source_spec_data['mode'] == modes[i] - if sum(ref_dupe_ind) and sum(date_dupe_ind) and sum(instrument_dupe_ind) and sum(mode_dupe_ind): + ref_dupe_ind = source_spec_data["reference"] == references[i] + date_dupe_ind = source_spec_data["observation_date"] == obs_date + instrument_dupe_ind = source_spec_data["instrument"] == instruments[i] + mode_dupe_ind = source_spec_data["mode"] == modes[i] + if ( + sum(ref_dupe_ind) + and sum(date_dupe_ind) + and sum(instrument_dupe_ind) + and sum(mode_dupe_ind) + ): msg = f"Skipping suspected duplicate measurement\n{source}\n" msg2 = f"{source_spec_data[ref_dupe_ind]['source', 'instrument', 'mode', 'observation_date', 'reference']}" msg3 = f"{instruments[i], modes[i], obs_date, references[i], spectra[i]} \n" @@ -1230,11 +1409,13 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes # n_skipped += 1 # continue if len(instrument) == 0 or len(mode) == 0 or len(telescope) == 0: - msg = f'Spectrum for {source} could not be added to the database. \n' \ - f' Telescope, Instrument, and/or Mode need to be added to the appropriate table. \n' \ - f" Trying to find telescope: {row_data[0]['telescope']}, instrument: {row_data[0]['instrument']}, " \ - f" mode: {row_data[0]['mode']} \n" \ - f" Telescope: {telescope}, Instrument: {instrument}, Mode: {mode} \n" + msg = ( + f"Spectrum for {source} could not be added to the database. \n" + f" Telescope, Instrument, and/or Mode need to be added to the appropriate table. \n" + f" Trying to find telescope: {row_data[0]['telescope']}, instrument: {row_data[0]['instrument']}, " + f" mode: {row_data[0]['mode']} \n" + f" Telescope: {telescope}, Instrument: {instrument}, Mode: {mode} \n" + ) logger.error(msg) n_missing_instrument += 1 if raise_error: @@ -1242,34 +1423,48 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes else: continue else: - msg = f'Spectrum for {source} could not be added to the database for unknown reason: \n {row_data} \n ' + msg = f"Spectrum for {source} could not be added to the database for unknown reason: \n {row_data} \n " logger.error(msg) raise SimpleError(msg) - msg = f"SPECTRA ADDED: {n_added} \n" \ - f" Spectra with blank obs_date: {n_blank} \n" \ - f" Suspected duplicates skipped: {n_dupes}\n" \ - f" Missing Telescope/Instrument/Mode: {n_missing_instrument} \n" \ - f" Spectra skipped for unknown reason: {n_skipped} \n" + msg = ( + f"SPECTRA ADDED: {n_added} \n" + f" Spectra with blank obs_date: {n_blank} \n" + f" Suspected duplicates skipped: {n_dupes}\n" + f" Missing Telescope/Instrument/Mode: {n_missing_instrument} \n" + f" Spectra skipped for unknown reason: {n_skipped} \n" + ) if n_spectra == 1: - logger.info(f"Added {source} : \n" - f"{row_data}") + logger.info(f"Added {source} : \n" f"{row_data}") else: logger.info(msg) - if n_added + n_dupes + n_blank + n_skipped + n_missing_instrument != n_spectra: msg = "Numbers don't add up: " logger.error(msg) raise SimpleError(msg) - spec_count = db.query(Spectra.regime, func.count(Spectra.regime)).group_by(Spectra.regime).all() - - spec_ref_count = db.query(Spectra.reference, func.count(Spectra.reference)). \ - group_by(Spectra.reference).order_by(func.count(Spectra.reference).desc()).limit(20).all() - - telescope_spec_count = db.query(Spectra.telescope, func.count(Spectra.telescope)). \ - group_by(Spectra.telescope).order_by(func.count(Spectra.telescope).desc()).limit(20).all() + spec_count = ( + db.query(Spectra.regime, func.count(Spectra.regime)) + .group_by(Spectra.regime) + .all() + ) + + spec_ref_count = ( + db.query(Spectra.reference, func.count(Spectra.reference)) + .group_by(Spectra.reference) + .order_by(func.count(Spectra.reference).desc()) + .limit(20) + .all() + ) + + telescope_spec_count = ( + db.query(Spectra.telescope, func.count(Spectra.telescope)) + .group_by(Spectra.telescope) + .order_by(func.count(Spectra.telescope).desc()) + .limit(20) + .all() + ) # logger.info(f'Spectra in the database: \n {spec_count} \n {spec_ref_count} \n {telescope_spec_count}') @@ -1302,49 +1497,66 @@ def ingest_instrument(db, telescope=None, instrument=None, mode=None): logger.error(msg) raise SimpleError(msg) - msg_search = f'Searching for {telescope}, {instrument}, {mode} in database' + msg_search = f"Searching for {telescope}, {instrument}, {mode} in database" logger.info(msg_search) # Search for the inputs in the database - telescope_db = db.query(db.Telescopes).filter(db.Telescopes.c.telescope == telescope).table() - mode_db = db.query(db.Instruments).filter(and_(db.Instruments.c.mode == mode, - db.Instruments.c.instrument == instrument, - db.Instruments.c.telescope == telescope)).table() + telescope_db = ( + db.query(db.Telescopes).filter(db.Telescopes.c.telescope == telescope).table() + ) + mode_db = ( + db.query(db.Instruments) + .filter( + and_( + db.Instruments.c.mode == mode, + db.Instruments.c.instrument == instrument, + db.Instruments.c.telescope == telescope, + ) + ) + .table() + ) if len(telescope_db) == 1 and len(mode_db) == 1: - msg_found = f'{telescope}, {instrument}, and {mode} are already in the database.' + msg_found = ( + f"{telescope}, {instrument}, and {mode} are already in the database." + ) logger.info(msg_found) return # Ingest telescope entry if not already present if telescope is not None and len(telescope_db) == 0: - telescope_add = [{'telescope': telescope}] + telescope_add = [{"telescope": telescope}] try: with db.engine.connect() as conn: conn.execute(db.Telescopes.insert().values(telescope_add)) conn.commit() - msg_telescope = f'{telescope} was successfully ingested in the database' + msg_telescope = f"{telescope} was successfully ingested in the database" logger.info(msg_telescope) except sqlalchemy.exc.IntegrityError as e: # pylint: disable=invalid-name - msg = 'Telescope could not be ingested' + msg = "Telescope could not be ingested" logger.error(msg) - raise SimpleError(msg + '\n' + str(e)) + raise SimpleError(msg + "\n" + str(e)) # Ingest instrument+mode (requires telescope) if not already present - if telescope is not None and instrument is not None and mode is not None and len(mode_db) == 0: - instrument_add = [{'instrument': instrument, - 'mode': mode, - 'telescope': telescope}] + if ( + telescope is not None + and instrument is not None + and mode is not None + and len(mode_db) == 0 + ): + instrument_add = [ + {"instrument": instrument, "mode": mode, "telescope": telescope} + ] try: with db.engine.connect() as conn: conn.execute(db.Instruments.insert().values(instrument_add)) conn.commit() - msg_instrument = f'{instrument} was successfully ingested in the database.' + msg_instrument = f"{instrument} was successfully ingested in the database." logger.info(msg_instrument) except sqlalchemy.exc.IntegrityError as e: # pylint: disable=invalid-name - msg = 'Instrument/Mode could not be ingested' + msg = "Instrument/Mode could not be ingested" logger.error(msg) - raise SimpleError(msg + '\n' + str(e)) + raise SimpleError(msg + "\n" + str(e)) return @@ -1364,13 +1576,15 @@ def get_gaiadr3(gaia_id, verbose=True): Table of Gaia data """ - gaia_query_string = f"SELECT " \ - f"parallax, parallax_error, " \ - f"pmra, pmra_error, pmdec, pmdec_error, " \ - f"phot_g_mean_flux, phot_g_mean_flux_error, phot_g_mean_mag, " \ - f"phot_rp_mean_flux, phot_rp_mean_flux_error, phot_rp_mean_mag " \ - f"FROM gaiadr3.gaia_source WHERE " \ - f"gaiadr3.gaia_source.source_id = '{gaia_id}'" + gaia_query_string = ( + f"SELECT " + f"parallax, parallax_error, " + f"pmra, pmra_error, pmdec, pmdec_error, " + f"phot_g_mean_flux, phot_g_mean_flux_error, phot_g_mean_mag, " + f"phot_rp_mean_flux, phot_rp_mean_flux_error, phot_rp_mean_mag " + f"FROM gaiadr3.gaia_source WHERE " + f"gaiadr3.gaia_source.source_id = '{gaia_id}'" + ) job_gaia_query = Gaia.launch_job(gaia_query_string, verbose=verbose) gaia_data = job_gaia_query.get_results() @@ -1380,57 +1594,91 @@ def get_gaiadr3(gaia_id, verbose=True): def ingest_gaia_photometry(db, sources, gaia_data, ref): # TODO write some tests - unmasked_gphot = np.logical_not(gaia_data['phot_g_mean_mag'].mask).nonzero() - gaia_g_phot = gaia_data[unmasked_gphot]['phot_g_mean_flux', 'phot_g_mean_flux_error', - 'phot_g_mean_mag'] + unmasked_gphot = np.logical_not(gaia_data["phot_g_mean_mag"].mask).nonzero() + gaia_g_phot = gaia_data[unmasked_gphot][ + "phot_g_mean_flux", "phot_g_mean_flux_error", "phot_g_mean_mag" + ] - unmased_rpphot = np.logical_not(gaia_data['phot_rp_mean_mag'].mask).nonzero() - gaia_rp_phot = gaia_data[unmased_rpphot]['phot_rp_mean_flux', 'phot_rp_mean_flux_error', - 'phot_rp_mean_mag'] + unmased_rpphot = np.logical_not(gaia_data["phot_rp_mean_mag"].mask).nonzero() + gaia_rp_phot = gaia_data[unmased_rpphot][ + "phot_rp_mean_flux", "phot_rp_mean_flux_error", "phot_rp_mean_mag" + ] # e_Gmag=abs(-2.5/ln(10)*e_FG/FG) from Vizier Note 37 on Gaia DR2 (I/345/gaia2) - gaia_g_phot['g_unc'] = np.abs( - -2.5 / np.log(10) * gaia_g_phot['phot_g_mean_flux_error'] / gaia_g_phot['phot_g_mean_flux']) - gaia_rp_phot['rp_unc'] = np.abs( - -2.5 / np.log(10) * gaia_rp_phot['phot_rp_mean_flux_error'] / gaia_rp_phot['phot_rp_mean_flux']) - - if ref == 'GaiaDR2': - g_band_name = 'GAIA2.G' - rp_band_name = 'GAIA2.Grp' - elif ref == 'GaiaEDR3' or ref == 'GaiaDR3': - g_band_name = 'GAIA3.G' - rp_band_name = 'GAIA3.Grp' + gaia_g_phot["g_unc"] = np.abs( + -2.5 + / np.log(10) + * gaia_g_phot["phot_g_mean_flux_error"] + / gaia_g_phot["phot_g_mean_flux"] + ) + gaia_rp_phot["rp_unc"] = np.abs( + -2.5 + / np.log(10) + * gaia_rp_phot["phot_rp_mean_flux_error"] + / gaia_rp_phot["phot_rp_mean_flux"] + ) + + if ref == "GaiaDR2": + g_band_name = "GAIA2.G" + rp_band_name = "GAIA2.Grp" + elif ref == "GaiaEDR3" or ref == "GaiaDR3": + g_band_name = "GAIA3.G" + rp_band_name = "GAIA3.Grp" else: raise Exception - ingest_photometry(db, sources, g_band_name, gaia_g_phot['phot_g_mean_mag'], gaia_g_phot['g_unc'], - ref, ucds='em.opt', telescope='Gaia', instrument='Gaia') - - ingest_photometry(db, sources, rp_band_name, gaia_rp_phot['phot_rp_mean_mag'], - gaia_rp_phot['rp_unc'], ref, ucds='em.opt.R', telescope='Gaia', instrument='Gaia') + ingest_photometry( + db, + sources, + g_band_name, + gaia_g_phot["phot_g_mean_mag"], + gaia_g_phot["g_unc"], + ref, + ucds="em.opt", + telescope="Gaia", + instrument="Gaia", + ) + + ingest_photometry( + db, + sources, + rp_band_name, + gaia_rp_phot["phot_rp_mean_mag"], + gaia_rp_phot["rp_unc"], + ref, + ucds="em.opt.R", + telescope="Gaia", + instrument="Gaia", + ) return def ingest_gaia_parallaxes(db, sources, gaia_data, ref): # TODO write some tests - unmasked_pi = np.logical_not(gaia_data['parallax'].mask).nonzero() - gaia_parallaxes = gaia_data[unmasked_pi]['parallax', 'parallax_error'] + unmasked_pi = np.logical_not(gaia_data["parallax"].mask).nonzero() + gaia_parallaxes = gaia_data[unmasked_pi]["parallax", "parallax_error"] - ingest_parallaxes(db, sources, gaia_parallaxes['parallax'], - gaia_parallaxes['parallax_error'], ref) + ingest_parallaxes( + db, sources, gaia_parallaxes["parallax"], gaia_parallaxes["parallax_error"], ref + ) def ingest_gaia_pms(db, sources, gaia_data, ref): # TODO write some tests - unmasked_pms = np.logical_not(gaia_data['pmra'].mask).nonzero() - pms = gaia_data[unmasked_pms]['pmra', 'pmra_error', 'pmdec', 'pmdec_error'] + unmasked_pms = np.logical_not(gaia_data["pmra"].mask).nonzero() + pms = gaia_data[unmasked_pms]["pmra", "pmra_error", "pmdec", "pmdec_error"] refs = [ref] * len(pms) - ingest_proper_motions(db, sources, - pms['pmra'], pms['pmra_error'], - pms['pmdec'], pms['pmdec_error'], - refs) + ingest_proper_motions( + db, + sources, + pms["pmra"], + pms["pmra_error"], + pms["pmdec"], + pms["pmdec_error"], + refs, + ) def ingest_spectrum_from_fits(db, source, spectrum_fits_file): @@ -1445,31 +1693,55 @@ def ingest_spectrum_from_fits(db, source, spectrum_fits_file): """ header = fits.getheader(spectrum_fits_file) - regime = header['SPECBAND'] - if regime == 'opt': - regime = 'optical' - telescope = header['TELESCOP'] - instrument = header['INSTRUME'] + regime = header["SPECBAND"] + if regime == "opt": + regime = "optical" + telescope = header["TELESCOP"] + instrument = header["INSTRUME"] try: - mode = header['MODE'] + mode = header["MODE"] except KeyError: mode = None - obs_date = header['DATE-OBS'] - doi = header['REFERENC'] + obs_date = header["DATE-OBS"] + doi = header["REFERENC"] data_header = fits.getheader(spectrum_fits_file, 1) - w_unit = data_header['TUNIT1'] - flux_unit = data_header['TUNIT2'] - - reference_match = db.query(db.Publications.c.publication).filter(db.Publications.c.doi == doi).table() - reference = reference_match['publication'][0] - - ingest_spectra(db, source, spectrum_fits_file, regime, telescope, instrument, None, obs_date, reference, - wavelength_units=w_unit, flux_units=flux_unit) - -#COMPANION RELATIONSHIP -def ingest_companion_relationships(db, source, companion_name, relationship, - projected_separation_arcsec = None, projected_separation_error = None, - comment = None, ref = None, other_companion_names = None): + w_unit = data_header["TUNIT1"] + flux_unit = data_header["TUNIT2"] + + reference_match = ( + db.query(db.Publications.c.publication) + .filter(db.Publications.c.doi == doi) + .table() + ) + reference = reference_match["publication"][0] + + ingest_spectra( + db, + source, + spectrum_fits_file, + regime, + telescope, + instrument, + None, + obs_date, + reference, + wavelength_units=w_unit, + flux_units=flux_unit, + ) + + +# COMPANION RELATIONSHIP +def ingest_companion_relationships( + db, + source, + companion_name, + relationship, + projected_separation_arcsec=None, + projected_separation_error=None, + comment=None, + ref=None, + other_companion_names=None, +): """ This function ingests a single row in to the CompanionRelationship table @@ -1479,7 +1751,7 @@ def ingest_companion_relationships(db, source, companion_name, relationship, Database object created by astrodbkit2 source: str Name of source as it appears in sources table - relationship: str + relationship: str relationship is of the souce to its companion should be one of the following: Child, Sibling, Parent, or Unresolved Parent see note @@ -1508,29 +1780,31 @@ def ingest_companion_relationships(db, source, companion_name, relationship, - *Unresolved Parent*: The source is the unresolved, combined light source of an unresolved multiple system which includes the companion - """ + """ # checking relationship entered - possible_relationships = ['Child', 'Sibling', 'Parent', 'Unresolved Parent', None] + possible_relationships = ["Child", "Sibling", "Parent", "Unresolved Parent", None] # check captialization if relationship.title() != relationship: - logger.info(f"Relationship captilization changed from {relationship} to {relationship.title()} ") + logger.info( + f"Relationship captilization changed from {relationship} to {relationship.title()} " + ) relationship = relationship.title() if relationship not in possible_relationships: msg = f"Relationship given for {source}, {companion_name}: {relationship} NOT one of the constrained relationships \n {possible_relationships}" logger.error(msg) raise SimpleError(msg) - + # source canot be same as companion if source == companion_name: msg = f"{source}: Source cannot be the same as companion name" logger.error(msg) raise SimpleError(msg) - + if source == companion_name: msg = f"{source}: Source cannot be the same as companion name" logger.error(msg) raise SimpleError(msg) - + if projected_separation_arcsec != None and projected_separation_arcsec < 0: msg = f"Projected separation: {projected_separation_arcsec}, cannot be negative" logger.error(msg) @@ -1541,42 +1815,286 @@ def ingest_companion_relationships(db, source, companion_name, relationship, raise SimpleError(msg) # check other names - ## make sure companion name is included in the list + # make sure companion name is included in the list if other_companion_names == None: other_companion_names = companion_name else: - companion_name_list = other_companion_names.split(', ') + companion_name_list = other_companion_names.split(", ") if companion_name not in companion_name_list: companion_name_list.append(companion_name) - other_companion_names = (', ').join(companion_name_list) - + other_companion_names = (", ").join(companion_name_list) try: with db.engine.connect() as conn: - conn.execute(db.CompanionRelationships.insert().values( - {'source': source, - 'companion_name': companion_name, - 'projected_separation_arcsec':projected_separation_arcsec, - 'projected_separation_error':projected_separation_error, - 'relationship':relationship, - 'reference': ref, - 'comments': comment, - 'other_companion_names': other_companion_names})) + conn.execute( + db.CompanionRelationships.insert().values( + { + "source": source, + "companion_name": companion_name, + "projected_separation_arcsec": projected_separation_arcsec, + "projected_separation_error": projected_separation_error, + "relationship": relationship, + "reference": ref, + "comments": comment, + "other_companion_names": other_companion_names, + } + ) + ) conn.commit() - logger.info(f"ComapnionRelationship added: ", - [source, companion_name, relationship, projected_separation_arcsec, \ - projected_separation_error, comment, ref]) + logger.info( + f"ComapnionRelationship added: ", + [ + source, + companion_name, + relationship, + projected_separation_arcsec, + projected_separation_error, + comment, + ref, + ], + ) except sqlalchemy.exc.IntegrityError as e: - if 'UNIQUE constraint failed:' in str(e): + if "UNIQUE constraint failed:" in str(e): msg = "The companion may be a duplicate." logger.error(msg) raise SimpleError(msg) - + else: - msg = ("Make sure all require parameters are provided. \\" - "Other possible errors: source may not exist in Sources table \\" \ - "or the reference may not exist in the Publications table. " ) + msg = ( + "Make sure all require parameters are provided. \\" + "Other possible errors: source may not exist in Sources table \\" + "or the reference may not exist in the Publications table. " + ) logger.error(msg) raise SimpleError(msg) +def ingest_source( + db, + source, + reference: str = None, + ra: float = None, + dec: float = None, + epoch: str = None, + equinox: str = None, + other_reference: str = None, + comment: str = None, + raise_error: bool = True, + search_db: bool = True, +): + """ + Parameters + ---------- + db: astrodbkit2.astrodb.Database + Database object created by astrodbkit2 + sources: str + Names of sources + references: str + Discovery references of sources + ras: float, optional + Right ascensions of sources. Decimal degrees. + decs: float, optional + Declinations of sources. Decimal degrees. + comments: string, optional + Comments + epochs: str, optional + Epochs of coordinates + equinoxes: str, optional + Equinoxes of coordinates + other_references: str + raise_error: bool, optional + True (default): Raise an error if a source cannot be ingested + False: Log a warning but skip sources which cannot be ingested + search_db: bool, optional + True (default): Search database to see if source is already ingested + False: Ingest source without searching the database + + Returns + ------- + + None + + """ + + if ra is None and dec is None: + coords_provided = False + else: + coords_provided = True + ra = ra + dec = dec + epoch = epoch + equinox = equinox + logger.debug(f"coords_provided:{coords_provided}") + + # Find out if source is already in database or not + if coords_provided and search_db: + logger.debug(f"Checking database for: {source} at ra: {ra}, dec: {dec}") + name_matches = find_source_in_db(db, source, ra=ra, dec=dec) + elif search_db: + logger.debug(f"Checking database for: {source}") + name_matches = find_source_in_db(db, source) + elif not search_db: + name_matches = [] + else: + name_matches = None + + logger.debug(f"Source matches in database: {name_matches}") + + # Source is already in database + # Checking out alternate names + if len(name_matches) == 1 and search_db: + # Figure out if source name provided is an alternate name + db_source_matches = db.search_object( + source, output_table="Sources", fuzzy_search=False + ) + + # Try to add alternate source name to Names table + if len(db_source_matches) == 0: + alt_names_data = [{"source": name_matches[0], "other_name": source}] + try: + with db.engine.connect() as conn: + conn.execute(db.Names.insert().values(alt_names_data)) + conn.commit() + logger.info(f" Name added to database: {alt_names_data}\n") + except sqlalchemy.exc.IntegrityError as e: + msg = f" Could not add {alt_names_data} to database" + logger.warning(msg) + if raise_error: + raise SimpleError(msg + "\n" + str(e)) + else: + return + + msg = f"Not ingesting {source}. Already in database as {name_matches[0]}. \n " + if raise_error: + raise SimpleError(msg) + else: + logger.info(msg) + return # Source is already in database, nothing new to ingest + + # Multiple source matches in the database so unable to ingest source + elif len(name_matches) > 1 and search_db: + msg1 = f" Not ingesting {source}." + msg = f" More than one match for {source}\n {name_matches}\n" + logger.warning(msg1 + msg) + if raise_error: + raise SimpleError(msg) + else: + return + + # No match in the database, INGEST! + elif len(name_matches) == 0 or not search_db: + # Make sure reference is provided and in References table + if reference is None or ma.is_masked(reference): + msg = f"Not ingesting {source}. Discovery reference is blank. \n" + logger.warning(msg) + if raise_error: + raise SimpleError(msg) + else: + return + + ref_check = find_publication(db, name=reference) + logger.debug(f"ref_check: {ref_check}") + + if ref_check is False: + msg = ( + f"Skipping: {source}. Discovery reference {reference} is not in Publications table. \n" + f"(Add it with ingest_publication function.)" + ) + logger.warning(msg) + if raise_error: + raise SimpleError(msg + msg2) + else: + return + + # Try to get coordinates from SIMBAD if they were not provided + if not coords_provided: + # Try to get coordinates from SIMBAD + simbad_result_table = Simbad.query_object(source) + + if simbad_result_table is None: + msg = f"Not ingesting {source}. Coordinates are needed and could not be retrieved from SIMBAD. \n" + logger.warning(msg) + if raise_error: + raise SimpleError(msg) + else: + return + # One SIMBAD match! Using those coordinates for source. + elif len(simbad_result_table) == 1: + simbad_coords = ( + simbad_result_table["RA"][0] + " " + simbad_result_table["DEC"][0] + ) + simbad_skycoord = SkyCoord(simbad_coords, unit=(u.hourangle, u.deg)) + ra = simbad_skycoord.to_string(style="decimal").split()[0] + dec = simbad_skycoord.to_string(style="decimal").split()[1] + epoch = "2000" # Default coordinates from SIMBAD are epoch 2000. + equinox = "J2000" # Default frame from SIMBAD is IRCS and J2000. + msg = f"Coordinates retrieved from SIMBAD {ra}, {dec}" + logger.debug(msg) + else: + msg = f"Not ingesting {source}. Coordinates are needed and could not be retrieved from SIMBAD. \n" + logger.warning(msg) + if raise_error: + raise SimpleError(msg) + else: + return + + # Just in case other conditionals not met + else: + msg = f"Unexpected condition encountered ingesting {source}" + logger.error(msg) + raise SimpleError(msg) + + logger.debug(f" Ingesting {source}.") + + # Construct data to be added + source_data = [ + { + "source": source, + "ra": ra, + "dec": dec, + "reference": reference, + "epoch": epoch, + "equinox": equinox, + "other_references": other_reference, + "comments": comment, + } + ] + names_data = [{"source": source, "other_name": source}] + + # Try to add the source to the database + try: + with db.engine.connect() as conn: + conn.execute(db.Sources.insert().values(source_data)) + conn.commit() + msg = f"Added {str(source_data)}" + logger.info(f"Added {source}") + logger.debug(msg) + except sqlalchemy.exc.IntegrityError: + msg = ( + f"Not ingesting {source}. Not sure why. \n" + "The reference may not exist in Publications table. \n" + "Add it with ingest_publication function. \n" + ) + msg2 = f" {str(source_data)} " + logger.warning(msg) + logger.debug(msg2) + if raise_error: + raise SimpleError(msg + msg2) + else: + return + + # Try to add the source name to the Names table + try: + with db.engine.connect() as conn: + conn.execute(db.Names.insert().values(names_data)) + conn.commit() + logger.debug(f" Name added to database: {names_data}\n") + except sqlalchemy.exc.IntegrityError: + msg = f" Could not add {names_data} to database" + logger.warning(msg) + if raise_error: + raise SimpleError(msg) + else: + return + + return diff --git a/scripts/ingests/utils.py b/scripts/ingests/utils.py index b926f08b5..205a8927b 100644 --- a/scripts/ingests/utils.py +++ b/scripts/ingests/utils.py @@ -21,20 +21,31 @@ import socket from scripts import REFERENCE_TABLES -warnings.filterwarnings("ignore", module='astroquery.simbad') -logger = logging.getLogger('SIMPLE') +__all__ = [ + "SimpleError", + "load_simpledb", + "find_source_in_db", + "find_publication", + "ingest_publication", + "check_internet_connection", +] + +warnings.filterwarnings("ignore", module="astroquery.simbad") +logger = logging.getLogger("SIMPLE") # Logger setup # This will stream all logger messages to the standard output and apply formatting for that logger.propagate = False # prevents duplicated logging messages -LOGFORMAT = logging.Formatter('%(asctime)s %(levelname)s: %(message)s', datefmt='%m/%d/%Y %I:%M:%S%p') +LOGFORMAT = logging.Formatter( + "%(asctime)s %(levelname)s: %(message)s", datefmt="%m/%d/%Y %I:%M:%S%p" +) ch = logging.StreamHandler(stream=sys.stdout) ch.setFormatter(LOGFORMAT) # To prevent duplicate handlers, only add if they haven't been set previously if not len(logger.handlers): logger.addHandler(ch) logger.setLevel(logging.INFO) - + class SimpleError(Exception): pass @@ -56,29 +67,43 @@ def load_simpledb(db_file, recreatedb=True, reference_tables=REFERENCE_TABLES): # Utility function to load the database db_file_path = Path(db_file) - db_connection_string = 'sqlite:///' + db_file + db_connection_string = "sqlite:///" + db_file if recreatedb and db_file_path.exists(): os.remove(db_file) # removes the current .db file if one already exists if not db_file_path.exists(): - try: # Use fancy in-memory database, if supported by astrodbkit2 - db = Database('sqlite://', reference_tables=REFERENCE_TABLES) # creates and connects to a temporary in-memory database - db.load_database('data/') # loads the data from the data files into the database + try: # Use fancy in-memory database, if supported by astrodbkit2 + db = Database( + "sqlite://", reference_tables=REFERENCE_TABLES + ) # creates and connects to a temporary in-memory database + db.load_database( + "data/" + ) # loads the data from the data files into the database db.dump_sqlite(db_file) # dump in-memory database to file - db = Database(db_connection_string, reference_tables=REFERENCE_TABLES) # replace database object with new file version + db = Database( + db_connection_string, reference_tables=REFERENCE_TABLES + ) # replace database object with new file version except RuntimeError: # use in-file database - create_database(db_connection_string) # creates empty database based on the simple schema - db = Database(db_connection_string, reference_tables=REFERENCE_TABLES) # connects to the empty database - db.load_database('data/') # loads the data from the data files into the database + create_database( + db_connection_string + ) # creates empty database based on the simple schema + db = Database( + db_connection_string, reference_tables=REFERENCE_TABLES + ) # connects to the empty database + db.load_database( + "data/" + ) # loads the data from the data files into the database else: - db = Database(db_connection_string, reference_tables=REFERENCE_TABLES) # if database already exists, connects to .db file + db = Database( + db_connection_string, reference_tables=REFERENCE_TABLES + ) # if database already exists, connects to .db file return db -def find_source_in_db(db, source, ra=None, dec=None, search_radius=60.): +def find_source_in_db(db, source, ra=None, dec=None, search_radius=60.0): """ Find a source in the database given a source name and optional coordinates. @@ -113,54 +138,68 @@ def find_source_in_db(db, source, ra=None, dec=None, search_radius=60.): source = source.strip() - logger.debug(f'{source}: Searching for match in database.') + logger.debug(f"{source}: Searching for match in database.") - db_name_matches = db.search_object(source, output_table='Sources', fuzzy_search=False, verbose=False) + db_name_matches = db.search_object( + source, output_table="Sources", fuzzy_search=False, verbose=False + ) # NO MATCHES # If no matches, try fuzzy search if len(db_name_matches) == 0: logger.debug(f"{source}: No name matches, trying fuzzy search") - db_name_matches = db.search_object(source, output_table='Sources', fuzzy_search=True, verbose=False) + db_name_matches = db.search_object( + source, output_table="Sources", fuzzy_search=True, verbose=False + ) # If still no matches, try to resolve the name with Simbad if len(db_name_matches) == 0: logger.debug(f"{source}: No name matches, trying Simbad search") - db_name_matches = db.search_object(source, resolve_simbad=True, fuzzy_search=False, verbose=False) + db_name_matches = db.search_object( + source, resolve_simbad=True, fuzzy_search=False, verbose=False + ) # if still no matches, try spatial search using coordinates, if provided if len(db_name_matches) == 0 and coords: - location = SkyCoord(ra, dec, frame='icrs', unit='deg') - radius = u.Quantity(search_radius, unit='arcsec') - logger.info(f"{source}: No Simbad match, trying coord search around {location.ra.degree}, {location.dec}") + location = SkyCoord(ra, dec, frame="icrs", unit="deg") + radius = u.Quantity(search_radius, unit="arcsec") + logger.info( + f"{source}: No Simbad match, trying coord search around {location.ra.degree}, {location.dec}" + ) db_name_matches = db.query_region(location, radius=radius) # If still no matches, try to get the coords from SIMBAD if len(db_name_matches) == 0: simbad_result_table = Simbad.query_object(source) if simbad_result_table is not None and len(simbad_result_table) == 1: - simbad_coords = simbad_result_table['RA'][0] + ' ' + simbad_result_table['DEC'][0] + simbad_coords = ( + simbad_result_table["RA"][0] + " " + simbad_result_table["DEC"][0] + ) simbad_skycoord = SkyCoord(simbad_coords, unit=(u.hourangle, u.deg)) - ra = simbad_skycoord.to_string(style='decimal').split()[0] - dec = simbad_skycoord.to_string(style='decimal').split()[1] + ra = simbad_skycoord.to_string(style="decimal").split()[0] + dec = simbad_skycoord.to_string(style="decimal").split()[1] msg = f"Coordinates retrieved from SIMBAD {ra}, {dec}" logger.debug(msg) # Search database around that coordinate - radius = u.Quantity(search_radius, unit='arcsec') + radius = u.Quantity(search_radius, unit="arcsec") + msg2 = ( + f"Finding SIMBAD matches around {simbad_skycoord} with radius {radius}" + ) + logger.debug(msg2) db_name_matches = db.query_region(simbad_skycoord, radius=radius) if len(db_name_matches) == 1: - db_names = db_name_matches['source'].tolist() - logger.debug(f'One match found for {source}: {db_names[0]}') + db_names = db_name_matches["source"].tolist() + logger.debug(f"One match found for {source}: {db_names[0]}") elif len(db_name_matches) > 1: - db_names = db_name_matches['source'].tolist() - logger.debug(f'More than match found for {source}: {db_names}') + db_names = db_name_matches["source"].tolist() + logger.debug(f"More than match found for {source}: {db_names}") # TODO: Find way for user to choose correct match elif len(db_name_matches) == 0: db_names = [] - logger.debug(f' {source}: No match found') + logger.debug(f" {source}: No match found") else: - raise SimpleError(f'Unexpected condition searching for {source}') + raise SimpleError(f"Unexpected condition searching for {source}") return db_names @@ -227,19 +266,25 @@ def find_publication(db, name: str = None, doi: str = None, bibcode: str = None) not_null_pub_filters.append(db.Publications.c.bibcode.ilike(bibcode)) pub_search_table = Table() if len(not_null_pub_filters) > 0: - pub_search_table = db.query(db.Publications).filter(or_(*not_null_pub_filters)).table() + pub_search_table = ( + db.query(db.Publications).filter(or_(*not_null_pub_filters)).table() + ) n_pubs_found = len(pub_search_table) if n_pubs_found == 1: - logger.info(f'Found {n_pubs_found} matching publications for ' - f"{name} or {doi} or {bibcode}: {pub_search_table['reference'].data}") + logger.info( + f"Found {n_pubs_found} matching publications for " + f"{name} or {doi} or {bibcode}: {pub_search_table['reference'].data}" + ) if logger.level <= 10: # debug pub_search_table.pprint_all() - return True, pub_search_table['reference'].data[0] + return True, pub_search_table["reference"].data[0] if n_pubs_found > 1: - logger.warning(f'Found {n_pubs_found} matching publications for {name} or {doi} or {bibcode}') + logger.warning( + f"Found {n_pubs_found} matching publications for {name} or {doi} or {bibcode}" + ) if logger.level <= 30: # warning pub_search_table.pprint_all() return False, n_pubs_found @@ -247,26 +292,31 @@ def find_publication(db, name: str = None, doi: str = None, bibcode: str = None) # If no matches found, search using first four characters of input name if n_pubs_found == 0 and name: shorter_name = name[:4] - logger.debug(f'No matching publications for {name}, Trying {shorter_name}.') - fuzzy_query_shorter_name = '%' + shorter_name + '%' - pub_search_table = db.query(db.Publications).filter( - db.Publications.c.reference.ilike(fuzzy_query_shorter_name)).table() + logger.debug(f"No matching publications for {name}, Trying {shorter_name}.") + fuzzy_query_shorter_name = "%" + shorter_name + "%" + pub_search_table = ( + db.query(db.Publications) + .filter(db.Publications.c.reference.ilike(fuzzy_query_shorter_name)) + .table() + ) n_pubs_found_short = len(pub_search_table) if n_pubs_found_short == 0: - logger.warning(f'No matching publications for {name} or {shorter_name}') - logger.warning('Use add_publication() to add it to the database.') + logger.warning(f"No matching publications for {name} or {shorter_name}") + logger.warning("Use add_publication() to add it to the database.") return False, 0 if n_pubs_found_short > 0: - logger.debug(f'Found {n_pubs_found_short} matching publications for {shorter_name}') + logger.debug( + f"Found {n_pubs_found_short} matching publications for {shorter_name}" + ) if logger.level == 10: # debug pub_search_table.pprint_all() # Try to find numbers in the reference which might be a date - dates = re.findall(r'\d+', name) + dates = re.findall(r"\d+", name) # try to find a two digit date if len(dates) == 0: - logger.debug(f'Could not find a date in {name}') + logger.debug(f"Could not find a date in {name}") two_digit_date = None elif len(dates) == 1: if len(dates[0]) == 4: @@ -274,29 +324,33 @@ def find_publication(db, name: str = None, doi: str = None, bibcode: str = None) elif len(dates[0]) == 2: two_digit_date = dates[0] else: - logger.debug(f'Could not find a two digit date using {dates}') + logger.debug(f"Could not find a two digit date using {dates}") two_digit_date = None else: - logger.debug(f'Could not find a two digit date using {dates}') + logger.debug(f"Could not find a two digit date using {dates}") two_digit_date = None if two_digit_date: - logger.debug(f'Trying to limit using {two_digit_date}') + logger.debug(f"Trying to limit using {two_digit_date}") n_pubs_found_short_date = 0 pubs_found_short_date = [] - for pub in pub_search_table['reference']: + for pub in pub_search_table["reference"]: if pub.find(two_digit_date) != -1: n_pubs_found_short_date += 1 pubs_found_short_date.append(pub) if n_pubs_found_short_date == 1: - logger.debug(f'Found {n_pubs_found_short_date} matching publications for ' - f'{name} using {shorter_name} and {two_digit_date}') - logger.debug(f'{pubs_found_short_date}') + logger.debug( + f"Found {n_pubs_found_short_date} matching publications for " + f"{name} using {shorter_name} and {two_digit_date}" + ) + logger.debug(f"{pubs_found_short_date}") return True, pubs_found_short_date[0] else: - logger.warning(f'Found {n_pubs_found_short_date} matching publications for ' - f'{name} using {shorter_name} and {two_digit_date}') - logger.warning(f'{pubs_found_short_date}') + logger.warning( + f"Found {n_pubs_found_short_date} matching publications for " + f"{name} using {shorter_name} and {two_digit_date}" + ) + logger.warning(f"{pubs_found_short_date}") return False, n_pubs_found_short_date else: return False, n_pubs_found_short @@ -306,8 +360,14 @@ def find_publication(db, name: str = None, doi: str = None, bibcode: str = None) return -def ingest_publication(db, doi: str = None, bibcode: str = None, publication: str = None, description: str = None, - ignore_ads: bool = False): +def ingest_publication( + db, + doi: str = None, + bibcode: str = None, + publication: str = None, + description: str = None, + ignore_ads: bool = False, +): """ Adds publication to the database using DOI or ADS Bibcode, including metadata found with ADS. @@ -335,14 +395,16 @@ def ingest_publication(db, doi: str = None, bibcode: str = None, publication: st """ if not (publication or doi or bibcode): - logger.error('Publication, DOI, or Bibcode is required input') + logger.error("Publication, DOI, or Bibcode is required input") return - ads.config.token = os.getenv('ADS_TOKEN') + ads.config.token = os.getenv("ADS_TOKEN") if not ads.config.token and (not publication and (not doi or not bibcode)): - logger.error("An ADS_TOKEN environment variable must be set in order to auto-populate the fields.\n" - "Without an ADS_TOKEN, name and bibcode or DOI must be set explicity.") + logger.error( + "An ADS_TOKEN environment variable must be set in order to auto-populate the fields.\n" + "Without an ADS_TOKEN, name and bibcode or DOI must be set explicity." + ) return if ads.config.token and not ignore_ads: @@ -352,7 +414,7 @@ def ingest_publication(db, doi: str = None, bibcode: str = None, publication: st logger.debug(f"Use ADS set to {use_ads}") if bibcode: - if 'arXiv' in bibcode: + if "arXiv" in bibcode: arxiv_id = bibcode bibcode = None else: @@ -360,21 +422,25 @@ def ingest_publication(db, doi: str = None, bibcode: str = None, publication: st else: arxiv_id = None - name_add, bibcode_add, doi_add = '', '', '' + name_add, bibcode_add, doi_add = "", "", "" # Search ADS uing a provided arxiv id if arxiv_id and use_ads: - arxiv_matches = ads.SearchQuery(q=arxiv_id, fl=['id', 'bibcode', 'title', 'first_author', 'year', 'doi']) + arxiv_matches = ads.SearchQuery( + q=arxiv_id, fl=["id", "bibcode", "title", "first_author", "year", "doi"] + ) arxiv_matches_list = list(arxiv_matches) if len(arxiv_matches_list) != 1: - logger.error('should only be one matching arxiv id') + logger.error("should only be one matching arxiv id") return if len(arxiv_matches_list) == 1: logger.debug(f"Publication found in ADS using arxiv id: , {arxiv_id}") article = arxiv_matches_list[0] - logger.debug(f"{article.first_author}, {article.year}, {article.bibcode}, {article.title}") + logger.debug( + f"{article.first_author}, {article.year}, {article.bibcode}, {article.title}" + ) if not publication: # generate the name if it was not provided - name_stub = article.first_author.replace(',', '').replace(' ', '') + name_stub = article.first_author.replace(",", "").replace(" ", "") name_add = name_stub[0:4] + article.year[-2:] else: name_add = publication @@ -389,19 +455,23 @@ def ingest_publication(db, doi: str = None, bibcode: str = None, publication: st # Search ADS using a provided DOI if doi and use_ads: - doi_matches = ads.SearchQuery(doi=doi, fl=['id', 'bibcode', 'title', 'first_author', 'year', 'doi']) + doi_matches = ads.SearchQuery( + doi=doi, fl=["id", "bibcode", "title", "first_author", "year", "doi"] + ) doi_matches_list = list(doi_matches) if len(doi_matches_list) != 1: - logger.error('should only be one matching DOI') + logger.error("should only be one matching DOI") return if len(doi_matches_list) == 1: logger.debug(f"Publication found in ADS using DOI: {doi}") using = doi article = doi_matches_list[0] - logger.debug(f"{article.first_author}, {article.year}, {article.bibcode}, {article.title}") + logger.debug( + f"{article.first_author}, {article.year}, {article.bibcode}, {article.title}" + ) if not publication: # generate the name if it was not provided - name_stub = article.first_author.replace(',', '').replace(' ', '') + name_stub = article.first_author.replace(",", "").replace(" ", "") name_add = name_stub[0:4] + article.year[-2:] else: name_add = publication @@ -414,25 +484,30 @@ def ingest_publication(db, doi: str = None, bibcode: str = None, publication: st doi_add = doi if bibcode and use_ads: - bibcode_matches = ads.SearchQuery(bibcode=bibcode, fl=['id', 'bibcode', 'title', 'first_author', 'year', 'doi']) + bibcode_matches = ads.SearchQuery( + bibcode=bibcode, + fl=["id", "bibcode", "title", "first_author", "year", "doi"], + ) bibcode_matches_list = list(bibcode_matches) if len(bibcode_matches_list) == 0: - logger.error('not a valid bibcode:' + str(bibcode)) - logger.error('nothing added') + logger.error("not a valid bibcode:" + str(bibcode)) + logger.error("nothing added") raise elif len(bibcode_matches_list) > 1: - logger.error('should only be one matching bibcode for:' + str(bibcode)) - logger.error('nothing added') + logger.error("should only be one matching bibcode for:" + str(bibcode)) + logger.error("nothing added") raise elif len(bibcode_matches_list) == 1: logger.debug("Publication found in ADS using bibcode: " + str(bibcode)) using = str(bibcode) article = bibcode_matches_list[0] - logger.debug(f"{article.first_author}, {article.year}, {article.bibcode}, {article.doi}, {article.title}") + logger.debug( + f"{article.first_author}, {article.year}, {article.bibcode}, {article.doi}, {article.title}" + ) if not publication: # generate the name if it was not provided - name_stub = article.first_author.replace(',', '').replace(' ', '') + name_stub = article.first_author.replace(",", "").replace(" ", "") name_add = name_stub[0:4] + article.year[-2:] else: name_add = publication @@ -449,33 +524,40 @@ def ingest_publication(db, doi: str = None, bibcode: str = None, publication: st if publication and not bibcode and not doi: name_add = publication - using = 'user input' + using = "user input" - new_ref = [{'reference': name_add, 'bibcode': bibcode_add, 'doi': doi_add, 'description': description}] + new_ref = [ + { + "reference": name_add, + "bibcode": bibcode_add, + "doi": doi_add, + "description": description, + } + ] try: with db.engine.connect() as conn: conn.execute(db.Publications.insert().values(new_ref)) conn.commit() - logger.info(f'Added {name_add} to Publications table using {using}') + logger.info(f"Added {name_add} to Publications table using {using}") except sqlalchemy.exc.IntegrityError as error: - msg = f"Not able to add {new_ref} to the database. " \ - f"It's possible that a similar publication already exists in database\n"\ - "Use find_publication function before adding a new record" + msg = ( + f"Not able to add {new_ref} to the database. " + f"It's possible that a similar publication already exists in database\n" + "Use find_publication function before adding a new record" + ) logger.error(msg) raise SimpleError(msg + str(error)) return - def check_internet_connection(): # get current IP address of system ipaddress = socket.gethostbyname(socket.gethostname()) # checking system IP is the same as "127.0.0.1" or not. - if ipaddress == "127.0.0.1": # no internet + if ipaddress == "127.0.0.1": # no internet return False, ipaddress else: return True, ipaddress - diff --git a/tests/test_data.py b/tests/test_data.py index f4976813e..22b48cda3 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -116,6 +116,10 @@ def test_discovery_references(db): t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() assert len(t) == 91, f'found {len(t)} discovery reference entries for {ref}' + ref = 'Roth' + t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() + assert len(t) == 83, f'found {len(t)} discovery reference entries for {ref}' + def test_proper_motion_refs(db): """ diff --git a/tests/test_integrity.py b/tests/test_integrity.py index 43b6f1eac..53ac7db76 100644 --- a/tests/test_integrity.py +++ b/tests/test_integrity.py @@ -5,7 +5,7 @@ import pytest from . import REFERENCE_TABLES -from sqlalchemy import func, select, except_ +from sqlalchemy import func, and_ # , select, except_ from simple.schema import * from astrodbkit2.astrodb import create_database, Database, or_ from astropy.table import unique @@ -14,8 +14,8 @@ from astrodbkit2.utils import _name_formatter -DB_NAME = 'temp.db' -DB_PATH = 'data' +DB_NAME = "temp.db" +DB_PATH = "data" # Load the database for use in individual tests @@ -26,20 +26,26 @@ def db(): if os.path.exists(DB_NAME): os.remove(DB_NAME) - connection_string = 'sqlite:///' + DB_NAME + connection_string = "sqlite:///" + DB_NAME create_database(connection_string) assert os.path.exists(DB_NAME) # Connect to the new database and confirm it has the Sources table db = Database(connection_string, reference_tables=REFERENCE_TABLES) assert db - assert 'source' in [c.name for c in db.Sources.columns] + assert "source" in [c.name for c in db.Sources.columns] # Load data into an in-memory sqlite database first, for performance - temp_db = Database('sqlite://', reference_tables=REFERENCE_TABLES) # creates and connects to a temporary in-memory database - temp_db.load_database(DB_PATH, verbose=False) # loads the data from the data files into the database + temp_db = Database( + "sqlite://", reference_tables=REFERENCE_TABLES + ) # creates and connects to a temporary in-memory database + temp_db.load_database( + DB_PATH, verbose=False + ) # loads the data from the data files into the database temp_db.dump_sqlite(DB_NAME) # dump in-memory database to file - db = Database('sqlite:///' + DB_NAME, reference_tables=REFERENCE_TABLES) # replace database object with new file version + db = Database( + "sqlite:///" + DB_NAME, reference_tables=REFERENCE_TABLES + ) # replace database object with new file version return db @@ -47,18 +53,26 @@ def db(): def test_reference_uniqueness(db): # Verify that all Publications.name values are unique t = db.query(db.Publications.c.reference).astropy() - assert len(t) == len(unique(t, keys='reference')), 'duplicated publications found' + assert len(t) == len(unique(t, keys="reference")), "duplicated publications found" # Verify that DOI are supplied - t = db.query(db.Publications.c.reference).filter(db.Publications.c.doi.is_(None)).astropy() + t = ( + db.query(db.Publications.c.reference) + .filter(db.Publications.c.doi.is_(None)) + .astropy() + ) if len(t) > 0: - print(f'\n{len(t)} publications lacking DOI:') + print(f"\n{len(t)} publications lacking DOI:") print(t) # Verify that Bibcodes are supplied - t = db.query(db.Publications.c.reference).filter(db.Publications.c.bibcode.is_(None)).astropy() + t = ( + db.query(db.Publications.c.reference) + .filter(db.Publications.c.bibcode.is_(None)) + .astropy() + ) if len(t) > 0: - print(f'\n{len(t)} publications lacking ADS bibcodes:') + print(f"\n{len(t)} publications lacking ADS bibcodes:") print(t) @@ -66,38 +80,60 @@ def test_references(db): # Verify that all data point to an existing Publication ref_list = [] - table_list = ['Sources', 'Photometry', 'Parallaxes', 'ProperMotions', 'Spectra'] + table_list = ["Sources", "Photometry", "Parallaxes", "ProperMotions", "Spectra"] for table in table_list: # Get list of unique references t = db.query(db.metadata.tables[table].c.reference).distinct().astropy() - ref_list = ref_list + t['reference'].tolist() + ref_list = ref_list + t["reference"].tolist() # Getting unique set ref_list = list(set(ref_list)) # Confirm that all are in Publications - t = db.query(db.Publications.c.reference).filter(db.Publications.c.reference.in_(ref_list)).astropy() - assert len(t) == len(ref_list), 'Some references were not matched' + t = ( + db.query(db.Publications.c.reference) + .filter(db.Publications.c.reference.in_(ref_list)) + .astropy() + ) + assert len(t) == len(ref_list), "Some references were not matched" # List out publications that have not been used - t = db.query(db.Publications.c.reference).filter(db.Publications.c.reference.notin_(ref_list)).astropy() - assert len(t) <= 606, f'{len(t)} unused references' + t = ( + db.query(db.Publications.c.reference) + .filter(db.Publications.c.reference.notin_(ref_list)) + .astropy() + ) + assert len(t) <= 606, f"{len(t)} unused references" def test_publications(db): # Find unused references in the Sources Table - # stm = except_(select([db.Publications.c.reference]), select([db.Sources.c.reference])) + # stm = except_(select([db.Publications.c.reference]), + # select([db.Sources.c.reference])) # result = db.session.execute(stm) # s = result.scalars().all() # assert len(s) == 720, f'found {len(s)} unused references' # Find references with no doi or bibcode - t = db.query(db.Publications.c.reference).filter( - or_(and_(db.Publications.c.doi.is_(None), db.Publications.c.bibcode.is_(None)), - and_(db.Publications.c.doi.is_(''), db.Publications.c.bibcode.is_(None)), - and_(db.Publications.c.doi.is_(None), db.Publications.c.bibcode.is_('')), - and_(db.Publications.c.doi.is_(''), db.Publications.c.bibcode.is_('')))).astropy() - assert len(t) == 28, f'found {len(t)} publications with missing bibcode and doi' + t = ( + db.query(db.Publications.c.reference) + .filter( + or_( + and_( + db.Publications.c.doi.is_(None), db.Publications.c.bibcode.is_(None) + ), + and_( + db.Publications.c.doi.is_(""), db.Publications.c.bibcode.is_(None) + ), + and_( + db.Publications.c.doi.is_(None), db.Publications.c.bibcode.is_("") + ), + and_(db.Publications.c.doi.is_(""), db.Publications.c.bibcode.is_("")), + ) + ) + .astropy() + ) + assert len(t) == 28, f"found {len(t)} publications with missing bibcode and doi" def test_parameters(db): @@ -106,17 +142,21 @@ def test_parameters(db): """ t = db.query(db.Parameters).astropy() - assert len(t) > 0, 'Parameters table is empty' + assert len(t) > 0, "Parameters table is empty" # Check usage of Parameters param_list = db.query(db.ModeledParameters.c.parameter).astropy() if len(param_list) > 0: # Get unique values - param_list = list(param_list['parameter']) + param_list = list(param_list["parameter"]) param_list = list(set(param_list)) - t = db.query(db.Parameters).filter(db.Parameters.c.parameter.notin_(param_list)).astropy() + t = ( + db.query(db.Parameters) + .filter(db.Parameters.c.parameter.notin_(param_list)) + .astropy() + ) if len(t) > 0: - print('The following parameters are not being used:') + print("The following parameters are not being used:") print(t) # Skipping actual assertion test # assert len(t) == 0, f'{len(t)} unused parameters' @@ -124,22 +164,35 @@ def test_parameters(db): def test_coordinates(db): # Verify that all sources have valid coordinates - t = db.query(db.Sources.c.source, db.Sources.c.ra, db.Sources.c.dec).filter( - or_(db.Sources.c.ra.is_(None), db.Sources.c.ra < 0, db.Sources.c.ra > 360, - db.Sources.c.dec.is_(None), db.Sources.c.dec < -90, db.Sources.c.dec > 90)).astropy() + t = ( + db.query(db.Sources.c.source, db.Sources.c.ra, db.Sources.c.dec) + .filter( + or_( + db.Sources.c.ra.is_(None), + db.Sources.c.ra < 0, + db.Sources.c.ra > 360, + db.Sources.c.dec.is_(None), + db.Sources.c.dec < -90, + db.Sources.c.dec > 90, + ) + ) + .astropy() + ) if len(t) > 0: - print(f'\n{len(t)} Sources failed coordinate checks') + print(f"\n{len(t)} Sources failed coordinate checks") print(t) - assert len(t) == 0, f'{len(t)} Sources failed coordinate checks' + assert len(t) == 0, f"{len(t)} Sources failed coordinate checks" def test_source_names(db): # Verify that all sources have at least one entry in Names table - sql_text = "SELECT Sources.source FROM Sources LEFT JOIN Names " \ - "ON Names.source=Sources.source WHERE Names.source IS NULL" - missing_names = db.sql_query(sql_text, fmt='astropy') + sql_text = ( + "SELECT Sources.source FROM Sources LEFT JOIN Names " + "ON Names.source=Sources.source WHERE Names.source IS NULL" + ) + missing_names = db.sql_query(sql_text, fmt="astropy") assert len(missing_names) == 0 @@ -150,13 +203,14 @@ def test_source_uniqueness(db): assert len(source_names) == len(unique_source_names) # Another method to find the duplicates - sql_text = "SELECT Sources.source FROM Sources GROUP BY source " \ - "HAVING (Count(*) > 1)" - duplicate_names = db.sql_query(sql_text, fmt='astropy') + sql_text = ( + "SELECT Sources.source FROM Sources GROUP BY source " "HAVING (Count(*) > 1)" + ) + duplicate_names = db.sql_query(sql_text, fmt="astropy") # if duplicate_names is non_zero, print out duplicate names if len(duplicate_names) > 0: - print(f'\n{len(duplicate_names)} duplicated names') + print(f"\n{len(duplicate_names)} duplicated names") print(duplicate_names) assert len(duplicate_names) == 0 @@ -165,51 +219,69 @@ def test_source_uniqueness(db): def test_names_table(db): # Verify that all Sources contain at least one entry in the Names table name_list = db.query(db.Sources.c.source).astropy() - name_list = name_list['source'].tolist() - source_name_counts = db.query(db.Names.c.source). \ - filter(db.Names.c.source.in_(name_list)). \ - distinct(). \ - count() - assert len(name_list) == source_name_counts, 'ERROR: There are Sources without entries in the Names table' - - # Verify that each Source contains an entry in Names with Names.source = Names.other_source - valid_name_counts = db.query(db.Names.c.source). \ - filter(db.Names.c.source == db.Names.c.other_name). \ - distinct(). \ - count() - - # If the number of valid names don't match the number of sources, then there are cases that are missing + name_list = name_list["source"].tolist() + source_name_counts = ( + db.query(db.Names.c.source) + .filter(db.Names.c.source.in_(name_list)) + .distinct() + .count() + ) + assert ( + len(name_list) == source_name_counts + ), "ERROR: There are Sources without entries in the Names table" + + # Verify that each Source contains an entry in + # Names with Names.source = Names.other_source + valid_name_counts = ( + db.query(db.Names.c.source) + .filter(db.Names.c.source == db.Names.c.other_name) + .distinct() + .count() + ) + + # If the number of valid names don't match the number of sources, + # then there are cases that are missing # The script below will gather them and print them out if len(name_list) != valid_name_counts: - # Create a temporary table that groups entries in the Names table by their source name + # Create a temporary table that groups entries in the + # Names table by their source name # with a column containing a concatenation of all known names - t = db.query(db.Names.c.source, - func.group_concat(db.Names.c.other_name).label('names')). \ - group_by(db.Names.c.source). \ - astropy() - - # Get the list of entries whose source name are not present in the 'other_names' column - # Then return the Names table results so we can see what the DB has for these entries - results = [row['source'] for row in t if row['source'] not in row['names'].split(',')] - print('\nEntries in Names without Names.source == Names.other_name:') - print(db.query(db.Names). - filter(db.Names.c.source.in_(results)). - astropy()) - - assert len(name_list) == valid_name_counts, \ - 'ERROR: There are entries in Names without Names.source == Names.other_name' + t = ( + db.query( + db.Names.c.source, + func.group_concat(db.Names.c.other_name).label("names"), + ) + .group_by(db.Names.c.source) + .astropy() + ) + + # Get the list of entries whose source name + # are not present in the 'other_names' column + # Then return the Names table results + # so we can see what the DB has for these entries + results = [ + row["source"] for row in t if row["source"] not in row["names"].split(",") + ] + print("\nEntries in Names without Names.source == Names.other_name:") + print(db.query(db.Names).filter(db.Names.c.source.in_(results)).astropy()) + + assert ( + len(name_list) == valid_name_counts + ), "ERROR: There are entries in Names without Names.source == Names.other_name" # Verify that there are no empty strings as other_names in Names - blank_names = db.query(db.Names).filter(db.Names.c.other_name == '').astropy() - assert len(blank_names) == 0, \ - 'ERROR: There are entries in Names which are empty strings' + blank_names = db.query(db.Names).filter(db.Names.c.other_name == "").astropy() + assert ( + len(blank_names) == 0 + ), "ERROR: There are entries in Names which are empty strings" def test_source_uniqueness2(db): # Verify that all Sources.source values are unique and find the duplicates - sql_text = "SELECT Sources.source FROM Sources GROUP BY source " \ - "HAVING (Count(*) > 1)" - duplicate_names = db.sql_query(sql_text, fmt='astropy') + sql_text = ( + "SELECT Sources.source FROM Sources GROUP BY source " "HAVING (Count(*) > 1)" + ) + duplicate_names = db.sql_query(sql_text, fmt="astropy") # if duplicate_names is non_zero, print out duplicate names assert len(duplicate_names) == 0 @@ -223,34 +295,41 @@ def test_source_simbad(db): name_list = [s[0] for s in results] # Add all IDS to the Simbad output as well as the user-provided id - Simbad.add_votable_fields('ids') - Simbad.add_votable_fields('typed_id') + Simbad.add_votable_fields("ids") + Simbad.add_votable_fields("typed_id") simbad_results = Simbad.query_objects(name_list) # Get a nicely formatted list of Simbad names for each input row duplicate_count = 0 - for row in simbad_results[['TYPED_ID', 'IDS']].iterrows(): + for row in simbad_results[["TYPED_ID", "IDS"]].iterrows(): try: name, ids = row[0].decode("utf-8"), row[1].decode("utf-8") except AttributeError: # Catch decoding error name, ids = row[0], row[1] - simbad_names = [_name_formatter(s) for s in ids.split('|') - if _name_formatter(s) != '' and _name_formatter(s) is not None] + simbad_names = [ + _name_formatter(s) + for s in ids.split("|") + if _name_formatter(s) != "" and _name_formatter(s) is not None + ] if len(simbad_names) == 0: - print(f'No Simbad names for {name}') + print(f"No Simbad names for {name}") continue # Examine DB for each input, displaying results when more than one source matches - t = db.search_object(simbad_names, output_table='Sources', fmt='astropy', fuzzy_search=False) + t = db.search_object( + simbad_names, output_table="Sources", fmt="astropy", fuzzy_search=False + ) if len(t) > 1: - print(f'Multiple matches for {name}: {simbad_names}') - print(db.query(db.Names).filter(db.Names.c.source.in_(t['source'])).astropy()) + print(f"Multiple matches for {name}: {simbad_names}") + print( + db.query(db.Names).filter(db.Names.c.source.in_(t["source"])).astropy() + ) duplicate_count += 1 - assert duplicate_count == 0, 'Duplicate sources identified via Simbad queries' + assert duplicate_count == 0, "Duplicate sources identified via Simbad queries" def test_photometry(db): @@ -258,21 +337,26 @@ def test_photometry(db): # Check that no negative magnitudes have been provided, # nor any that are larger than 99 (if missing/limits, just use None) - t = db.query(db.Photometry). \ - filter(or_(db.Photometry.c.magnitude < 0, - db.Photometry.c.magnitude >= 99)). \ - astropy() + t = ( + db.query(db.Photometry) + .filter(or_(db.Photometry.c.magnitude < 0, db.Photometry.c.magnitude >= 99)) + .astropy() + ) if len(t) > 0: - print('\nInvalid magnitudes present') + print("\nInvalid magnitudes present") print(t) assert len(t) == 0 def test_photometry_filters(db): bands_in_use = db.query(db.Photometry.c.band).distinct().astropy() - for band_in_use in bands_in_use['band']: - check = db.query(db.PhotometryFilters).filter(db.PhotometryFilters.c.band == band_in_use).astropy() - assert len(check) == 1, f'{band_in_use} not in PhotometryFilters' + for band_in_use in bands_in_use["band"]: + check = ( + db.query(db.PhotometryFilters) + .filter(db.PhotometryFilters.c.band == band_in_use) + .astropy() + ) + assert len(check) == 1, f"{band_in_use} not in PhotometryFilters" def test_parallaxes(db): @@ -280,11 +364,15 @@ def test_parallaxes(db): # While there may be many parallax measurements for a single source, # there should be only one marked as adopted - t = db.query(db.Parallaxes.c.source, - func.sum(db.Parallaxes.c.adopted).label('adopted_counts')). \ - group_by(db.Parallaxes.c.source). \ - having(func.sum(db.Parallaxes.c.adopted) > 1). \ - astropy() + t = ( + db.query( + db.Parallaxes.c.source, + func.sum(db.Parallaxes.c.adopted).label("adopted_counts"), + ) + .group_by(db.Parallaxes.c.source) + .having(func.sum(db.Parallaxes.c.adopted) > 1) + .astropy() + ) if len(t) > 0: print("\nParallax entries with incorrect 'adopted' labels") print(t) @@ -295,22 +383,29 @@ def test_propermotions(db): # Tests against the ProperMotions table # There should be no entries in the ProperMotions table without both mu_ra and mu_dec - t = db.query(db.ProperMotions.c.source). \ - filter(or_(db.ProperMotions.c.mu_ra.is_(None), - db.ProperMotions.c.mu_dec.is_(None))). \ - astropy() + t = ( + db.query(db.ProperMotions.c.source) + .filter( + or_(db.ProperMotions.c.mu_ra.is_(None), db.ProperMotions.c.mu_dec.is_(None)) + ) + .astropy() + ) if len(t) > 0: - print('\nEntries found without proper motion values') + print("\nEntries found without proper motion values") print(t) assert len(t) == 0 # While there may be many proper motion measurements for a single source, # there should be only one marked as adopted - t = db.query(db.ProperMotions.c.source, - func.sum(db.ProperMotions.c.adopted).label('adopted_counts')). \ - group_by(db.ProperMotions.c.source). \ - having(func.sum(db.ProperMotions.c.adopted) > 1). \ - astropy() + t = ( + db.query( + db.ProperMotions.c.source, + func.sum(db.ProperMotions.c.adopted).label("adopted_counts"), + ) + .group_by(db.ProperMotions.c.source) + .having(func.sum(db.ProperMotions.c.adopted) > 1) + .astropy() + ) if len(t) > 0: print("\nProper Motion measurements with incorrect 'adopted' labels") print(t) @@ -321,21 +416,27 @@ def test_radialvelocities(db): # Tests against the RadialVelocities table # There should be no entries in the RadialVelocities table without rv values - t = db.query(db.RadialVelocities.c.source). \ - filter(db.RadialVelocities.c.radial_velocity.is_(None)). \ - astropy() + t = ( + db.query(db.RadialVelocities.c.source) + .filter(db.RadialVelocities.c.radial_velocity.is_(None)) + .astropy() + ) if len(t) > 0: - print('\nEntries found without radial velocity values') + print("\nEntries found without radial velocity values") print(t) assert len(t) == 0 # While there may be many radial velocity measurements for a single source, # there should be only one marked as adopted - t = db.query(db.RadialVelocities.c.source, - func.sum(db.RadialVelocities.c.adopted).label('adopted_counts')). \ - group_by(db.RadialVelocities.c.source). \ - having(func.sum(db.RadialVelocities.c.adopted) > 1). \ - astropy() + t = ( + db.query( + db.RadialVelocities.c.source, + func.sum(db.RadialVelocities.c.adopted).label("adopted_counts"), + ) + .group_by(db.RadialVelocities.c.source) + .having(func.sum(db.RadialVelocities.c.adopted) > 1) + .astropy() + ) if len(t) > 0: print("\nRadial velocity measurements with incorrect 'adopted' labels") print(t) @@ -346,30 +447,38 @@ def test_spectraltypes(db): # Tests against the SpectralTypes table # There should be no entries in the SpectralTypes table without a spectral type string - t = db.query(db.SpectralTypes.c.source). \ - filter(db.SpectralTypes.c.spectral_type_string.is_(None)). \ - astropy() + t = ( + db.query(db.SpectralTypes.c.source) + .filter(db.SpectralTypes.c.spectral_type_string.is_(None)) + .astropy() + ) if len(t) > 0: - print('\nEntries found without spectral type strings') + print("\nEntries found without spectral type strings") print(t) assert len(t) == 0 # There should be no entries in the SpectralTypes table without a spectral type code - t = db.query(db.SpectralTypes.c.source). \ - filter(db.SpectralTypes.c.spectral_type_code.is_(None)). \ - astropy() + t = ( + db.query(db.SpectralTypes.c.source) + .filter(db.SpectralTypes.c.spectral_type_code.is_(None)) + .astropy() + ) if len(t) > 0: - print('\nEntries found without spectral type codes') + print("\nEntries found without spectral type codes") print(t) assert len(t) == 0 # While there may be many spectral type measurements for a single source, # there should be only one marked as adopted - t = db.query(db.SpectralTypes.c.source, - func.sum(db.SpectralTypes.c.adopted).label('adopted_counts')). \ - group_by(db.SpectralTypes.c.source). \ - having(func.sum(db.SpectralTypes.c.adopted) > 1). \ - astropy() + t = ( + db.query( + db.SpectralTypes.c.source, + func.sum(db.SpectralTypes.c.adopted).label("adopted_counts"), + ) + .group_by(db.SpectralTypes.c.source) + .having(func.sum(db.SpectralTypes.c.adopted) > 1) + .astropy() + ) if len(t) > 0: print("\nSpectral Type entries with incorrect 'adopted' labels") print(t) @@ -380,139 +489,175 @@ def test_gravities(db): # Tests against the Gravities table # There should be no entries in the Gravities table without a gravity measurement - t = db.query(db.Gravities.c.source). \ - filter(db.Gravities.c.gravity.is_(None)). \ - astropy() + t = ( + db.query(db.Gravities.c.source) + .filter(db.Gravities.c.gravity.is_(None)) + .astropy() + ) if len(t) > 0: - print('\nEntries found without gravity values') + print("\nEntries found without gravity values") print(t) assert len(t) == 0 def test_sources(db): # Counting the top 20 references in the Sources Table - spec_ref_count = db.query(Sources.reference, func.count(Sources.reference)). \ - group_by(Sources.reference).order_by(func.count(Sources.reference).desc()).limit(20).all() + # spec_ref_count = ( + # db.query(Sources.reference, func.count(Sources.reference)) + # .group_by(Sources.reference) + # .order_by(func.count(Sources.reference).desc()) + # .limit(20) + # .all() + # ) # Top 20 References in the Sources Table - ref = 'Schm10.1808' + ref = "Schm10.1808" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 208, f'found {len(t)} sources from {ref}' + assert len(t) == 208, f"found {len(t)} sources from {ref}" - ref = 'Reid08.1290' + ref = "Reid08.1290" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 206, f'found {len(t)} sources from {ref}' + assert len(t) == 206, f"found {len(t)} sources from {ref}" - ref = 'West08' + ref = "West08" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 192, f'found {len(t)} sources from {ref}' + assert len(t) == 192, f"found {len(t)} sources from {ref}" - ref = 'Cruz03' + ref = "Cruz03" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 165, f'found {len(t)} sources from {ref}' + assert len(t) == 165, f"found {len(t)} sources from {ref}" - ref = 'Maro15' + ref = "Maro15" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 113, f'found {len(t)} sources from {ref}' + assert len(t) == 113, f"found {len(t)} sources from {ref}" - ref = 'Best15' + ref = "Best15" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 101, f'found {len(t)} sources from {ref}' + assert len(t) == 101, f"found {len(t)} sources from {ref}" - ref = 'Kirk11' + ref = "Kirk11" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 100, f'found {len(t)} sources from {ref}' + assert len(t) == 100, f"found {len(t)} sources from {ref}" - ref = 'Mace13.6' + ref = "Mace13.6" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 93, f'found {len(t)} sources from {ref}' + assert len(t) == 93, f"found {len(t)} sources from {ref}" - ref = 'Cruz07' + ref = "Cruz07" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 91, f'found {len(t)} sources from {ref}' + assert len(t) == 91, f"found {len(t)} sources from {ref}" - ref = 'Burn13' + ref = "Burn13" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 69, f'found {len(t)} sources from {ref}' + assert len(t) == 69, f"found {len(t)} sources from {ref}" - ref = 'Gagn15.33' + ref = "Gagn15.33" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 68, f'found {len(t)} sources from {ref}' + assert len(t) == 68, f"found {len(t)} sources from {ref}" - ref = 'Chiu06' + ref = "Chiu06" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 62, f'found {len(t)} sources from {ref}' + assert len(t) == 62, f"found {len(t)} sources from {ref}" - ref = 'Kirk00' + ref = "Kirk00" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 61, f'found {len(t)} sources from {ref}' + assert len(t) == 61, f"found {len(t)} sources from {ref}" - ref = 'DayJ13' + ref = "DayJ13" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 61, f'found {len(t)} sources from {ref}' + assert len(t) == 61, f"found {len(t)} sources from {ref}" - ref = 'Kirk10' + ref = "Kirk10" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 56, f'found {len(t)} sources from {ref}' + assert len(t) == 56, f"found {len(t)} sources from {ref}" - ref = 'Deac14.119' + ref = "Deac14.119" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 52, f'found {len(t)} sources from {ref}' + assert len(t) == 52, f"found {len(t)} sources from {ref}" - ref = 'Hawl02' + ref = "Hawl02" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 51, f'found {len(t)} sources from {ref}' + assert len(t) == 51, f"found {len(t)} sources from {ref}" - ref = 'Card15' + ref = "Card15" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 45, f'found {len(t)} sources from {ref}' + assert len(t) == 45, f"found {len(t)} sources from {ref}" - ref = 'Burn10.1885' + ref = "Burn10.1885" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 43, f'found {len(t)} sources from {ref}' + assert len(t) == 43, f"found {len(t)} sources from {ref}" - ref = 'Albe11' + ref = "Albe11" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 37, f'found {len(t)} sources from {ref}' + assert len(t) == 37, f"found {len(t)} sources from {ref}" def test_modeled_parameters(db): # There should be no entries in the modeled parameters table without parameter - t = db.query(db.ModeledParameters).filter(db.ModeledParameters.c.parameter.is_(None)).astropy() + t = ( + db.query(db.ModeledParameters) + .filter(db.ModeledParameters.c.parameter.is_(None)) + .astropy() + ) if len(t) > 0: - print('\nEntries found without a parameter') + print("\nEntries found without a parameter") print(t) assert len(t) == 0 # Test units are astropy.unit resolvable - t = db.query(db.ModeledParameters).filter(db.ModeledParameters.c.unit.is_not(None)).distinct().astropy() + t = ( + db.query(db.ModeledParameters) + .filter(db.ModeledParameters.c.unit.is_not(None)) + .distinct() + .astropy() + ) unit_fail = [] for x in t: - unit = x['unit'] + unit = x["unit"] try: - assert u.Unit(unit, parse_strict='raise') + assert u.Unit(unit, parse_strict="raise") except ValueError: - print(f'{unit} is not a recognized astropy unit') - counts = db.query(db.ModeledParameters).filter(db.ModeledParameters.c.unit == unit).count() + print(f"{unit} is not a recognized astropy unit") + counts = ( + db.query(db.ModeledParameters) + .filter(db.ModeledParameters.c.unit == unit) + .count() + ) unit_fail.append({unit: counts}) # count of how many of that unit there is - assert len(unit_fail) == 0, f'Some parameter units did not resolve: {unit_fail}' + assert len(unit_fail) == 0, f"Some parameter units did not resolve: {unit_fail}" # check no negative Mass, Radius, or Teff - t = db.query(db.ModeledParameters).filter(and_(db.ModeledParameters.c.parameter.in_(["radius", "mass", "Teff"]), - db.ModeledParameters.c.value < 0)).astropy() + t = ( + db.query(db.ModeledParameters) + .filter( + and_( + db.ModeledParameters.c.parameter.in_(["radius", "mass", "Teff"]), + db.ModeledParameters.c.value < 0, + ) + ) + .astropy() + ) if len(t) > 0: - print('\n Negative value for Radius, Mass, or Teff not allowed.\n') + print("\n Negative value for Radius, Mass, or Teff not allowed.\n") print(t) assert len(t) == 0 - # check no negative value error - t = db.query(db.ModeledParameters).filter(and_(db.ModeledParameters.c.value_error != None, - db.ModeledParameters.c.value_error < 0)).astropy() - + # check no negative value error + t = ( + db.query(db.ModeledParameters) + .filter( + and_( + db.ModeledParameters.c.value_error is not None, + db.ModeledParameters.c.value_error < 0, + ) + ) + .astropy() + ) + if len(t) > 0: - print('\n Negative projected separations') + print("\n Negative projected separations") print(t) assert len(t) == 0 @@ -521,11 +666,9 @@ def test_spectra(db): # Tests against the Spectra table # There should be no entries in the Spectra table without a spectrum - t = db.query(db.Spectra.c.source). \ - filter(db.Spectra.c.spectrum.is_(None)). \ - astropy() + t = db.query(db.Spectra.c.source).filter(db.Spectra.c.spectrum.is_(None)).astropy() if len(t) > 0: - print('\nEntries found without spectrum') + print("\nEntries found without spectrum") print(t) assert len(t) == 0 @@ -533,24 +676,48 @@ def test_spectra(db): def test_special_characters(db): # This test asserts that no special unicode characters are in the database # This can be expanded with additional characters we want to avoid - bad_characters = ['\u2013', '\u00f3', '\u00e9', '\u00ed', '\u00e1', '\u00fa'] + bad_characters = [ + "\u2013", + "\u00f3", + "\u00e9", + "\u00ed", + "\u00e1", + "\u00fa", + "\u0000" + ] for char in bad_characters: data = db.search_string(char) - # Check tables individually, want to make sure primary/foreign keys are verified but not comments/descriptions + # Make sure primary/foreign keys don't have unicode + # but not checking comments/descriptions if len(data) > 0: for table_name in data.keys(): - if table_name == 'Publications': - check = [char not in data[table_name]['reference']] - assert all(check), f'{char} in {table_name}' - elif table_name == 'Spectra': - check = [char not in data[table_name]['spectrum']] - assert all(check), f'{char} in {table_name}' - elif table_name == 'Names': - check = [char not in data[table_name]['other_name']] - assert all(check), f'{char} in {table_name}' + if table_name == "Publications": + check = [char not in data[table_name]["reference"]] + assert all(check), f"{char} in {table_name}" + elif table_name == "Spectra": + check = [char not in data[table_name]["spectrum"]] + assert all(check), f"{char} in {table_name}" + elif table_name == "Names": + check = [char not in data[table_name]["other_name"]] + assert all(check), f"{char} in {table_name}" + elif table_name == "Instruments": + check = [char not in data[table_name]["instrument"]] + assert all(check), f"{char} in {table_name}" + elif table_name == "Telescopes": + check = [char not in data[table_name]["telescope"]] + assert all(check), f"{char} in {table_name}" + elif table_name == "Parameters": + check = [char not in data[table_name]["parameter"]] + assert all(check), f"{char} in {table_name}" + elif table_name == "PhotometryFilters": + check = [char not in data[table_name]["band"]] + assert all(check), f"{char} in {table_name}" + elif table_name == "Versions": + check = [char not in data[table_name]["version"]] + assert all(check), f"{char} in {table_name}" else: - check = [char not in data[table_name]['source']] - assert all(check), f'{char} in {table_name}' + check = [char not in data[table_name]["source"]] + assert all(check), f"{char} in {table_name}" def test_database_views(db): @@ -563,93 +730,124 @@ def test_database_views(db): assert len(t) > 0 # Check view is not part of inventory - assert 'ParallaxView' not in db.inventory('2MASSI J0019457+521317').keys() - + assert "ParallaxView" not in db.inventory("2MASSI J0019457+521317").keys() + + def test_companion_relationship(db): # There should be no entries without a companion name - t = db.query(db.CompanionRelationships.c.source). \ - filter(db.CompanionRelationships.c.companion_name.is_(None)). \ - astropy() + t = ( + db.query(db.CompanionRelationships.c.source) + .filter(db.CompanionRelationships.c.companion_name.is_(None)) + .astropy() + ) if len(t) > 0: - print('\n Entries found without a companion name') + print("\n Entries found without a companion name") print(t) assert len(t) == 0 # There should be no entries a companion name thats the same as the source - t = db.query(db.CompanionRelationships.c.source). \ - filter(db.CompanionRelationships.c.companion_name == db.CompanionRelationships.c.source). \ - astropy() + t = ( + db.query(db.CompanionRelationships.c.source) + .filter( + db.CompanionRelationships.c.companion_name + == db.CompanionRelationships.c.source + ) + .astropy() + ) if len(t) > 0: - print('\nCompanion name cannot be source name') + print("\nCompanion name cannot be source name") print(t) assert len(t) == 0 # check no negative separations or error - ## first separtation - t = db.query(db.CompanionRelationships). \ - filter(and_(db.CompanionRelationships.c.projected_separation_arcsec != None, - db.CompanionRelationships.c.projected_separation_arcsec < 0)).astropy() + # first separtation + t = ( + db.query(db.CompanionRelationships) + .filter( + and_( + db.CompanionRelationships.c.projected_separation_arcsec is not None, + db.CompanionRelationships.c.projected_separation_arcsec < 0, + ) + ) + .astropy() + ) if len(t) > 0: - print('\n Negative projected separations') + print("\n Negative projected separations") print(t) assert len(t) == 0 - ## separation error - t = db.query(db.CompanionRelationships). \ - filter(and_(db.CompanionRelationships.c.projected_separation_error != None, - db.CompanionRelationships.c.projected_separation_error < 0)). \ - astropy() - + # separation error + t = ( + db.query(db.CompanionRelationships) + .filter( + and_( + db.CompanionRelationships.c.projected_separation_error is not None, + db.CompanionRelationships.c.projected_separation_error < 0, + ) + ) + .astropy() + ) + if len(t) > 0: - print('\n Negative projected separations') + print("\n Negative projected separations") print(t) assert len(t) == 0 - # test correct relationship - possible_relationships = ['Child', 'Sibling', 'Parent', 'Unresolved Parent'] - t = db.query(db.CompanionRelationships). \ - filter(~db.CompanionRelationships.c.relationship.in_(possible_relationships)). \ - astropy() + # test correct relationship + possible_relationships = ["Child", "Sibling", "Parent", "Unresolved Parent"] + t = ( + db.query(db.CompanionRelationships) + .filter(~db.CompanionRelationships.c.relationship.in_(possible_relationships)) + .astropy() + ) if len(t) > 0: - print('\n relationship is of the souce to its companion \ - should be one of the following: Child, Sibling, Parent, or Unresolved Parent') + print( + "\n relationship is of the souce to its companion \ + should be one of the following: Child, Sibling, Parent, or Unresolved Parent" + ) print(t) assert len(t) == 0 def test_companion_relationship_uniqueness(db): # Verify that all souces and companion_names values are unique combinations - ## first finding duplicate sources - sql_text = "SELECT CompanionRelationships.source FROM CompanionRelationships GROUP BY source " \ - "HAVING (Count(*) > 1)" - duplicate_sources = db.sql_query(sql_text, fmt='astropy') - - ##checking duplicate sources have different companions + # first finding duplicate sources + sql_text = ( + "SELECT CompanionRelationships.source " + "FROM CompanionRelationships GROUP BY source " + "HAVING (Count(*) > 1)" + ) + duplicate_sources = db.sql_query(sql_text, fmt="astropy") + + # checking duplicate sources have different companions non_unique = [] for source in duplicate_sources: t = db.query(db.CompanionRelationships.c.companion_name) - filter(db.CompanionRelationships.c.source == source). \ - astropy() - duplicate_companions = [n for n, companion in enumerate(t) if companion in t[:n]] + filter(db.CompanionRelationships.c.source == source).astropy() + duplicate_companions = [ + n for n, companion in enumerate(t) if companion in t[:n] + ] if len(duplicate_companions) > 0: non_unique.append(f"{source} and {duplicate_companions}") if len(non_unique) > 0: - print('\n Non-unique companion combination(s)') + print("\n Non-unique companion combination(s)") print(non_unique) assert len(non_unique) == 0 -def test_names_uniqueness(db): +def test_names_uniqueness(db): # Verify that all Names.other_name values are unique - sql_text = "SELECT Names.other_name FROM Names GROUP BY other_name " \ - "HAVING (Count(*) > 1)" - duplicate_names = db.sql_query(sql_text, fmt='astropy') + sql_text = ( + "SELECT Names.other_name FROM Names GROUP BY other_name " + "HAVING (Count(*) > 1)" + ) + duplicate_names = db.sql_query(sql_text, fmt="astropy") # if duplicate_names is non_zero, print out duplicate other names if len(duplicate_names) > 0: - print(f'\n{len(duplicate_names)} possibly a duplicated other_name.') + print(f"\n{len(duplicate_names)} possibly a duplicated other_name.") print(duplicate_names) assert len(duplicate_names) == 0 diff --git a/tests/test_utils.py b/tests/test_utils.py index 347f995d3..92a96a53b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,21 +1,34 @@ -# Test to verify functions in utils import pytest -import sys import math - -# pylint: disable=all - -sys.path.append('.') -from scripts.ingests.utils import * -from scripts.ingests.ingest_utils import * -from simple.schema import * +import os from astrodbkit2.astrodb import create_database, Database from astropy.table import Table from sqlalchemy import and_ +from scripts.ingests.utils import ( + SimpleError, + find_publication, + ingest_publication, +) +from scripts.ingests.ingest_utils import ( + convert_spt_string_to_code, + ingest_companion_relationships, + ingest_source, + ingest_sources, + ingest_parallaxes, + ingest_spectral_types, + ingest_instrument, + ingest_proper_motions, +) +from simple.schema import * +import logging -DB_NAME = 'temp.db' -DB_PATH = 'data' -logger.setLevel(logging.INFO) + +logger = logging.getLogger("SIMPLE") +logger.setLevel(logging.DEBUG) + + +DB_NAME = "temp.db" +DB_PATH = "data" # Load the database for use in individual tests @@ -26,14 +39,14 @@ def db(): if os.path.exists(DB_NAME): os.remove(DB_NAME) - connection_string = 'sqlite:///' + DB_NAME + connection_string = "sqlite:///" + DB_NAME create_database(connection_string) assert os.path.exists(DB_NAME) # Connect to the new database and confirm it has the Sources table db = Database(connection_string) assert db - assert 'source' in [c.name for c in db.Sources.columns] + assert "source" in [c.name for c in db.Sources.columns] return db @@ -41,10 +54,13 @@ def db(): # Create fake astropy Table of data to load @pytest.fixture(scope="module") def t_plx(): - t_plx = Table([{'source': 'Fake 1', 'plx': 113., 'plx_err': 0.3, 'plx_ref': 'Ref 1'}, - {'source': 'Fake 2', 'plx': 145., 'plx_err': 0.5, 'plx_ref': 'Ref 1'}, - {'source': 'Fake 3', 'plx': 155., 'plx_err': 0.6, 'plx_ref': 'Ref 2'}, - ]) + t_plx = Table( + [ + {"source": "Fake 1", "plx": 113.0, "plx_err": 0.3, "plx_ref": "Ref 1"}, + {"source": "Fake 2", "plx": 145.0, "plx_err": 0.5, "plx_ref": "Ref 1"}, + {"source": "Fake 3", "plx": 155.0, "plx_err": 0.6, "plx_ref": "Ref 2"}, + ] + ) return t_plx @@ -52,23 +68,53 @@ def t_plx(): @pytest.fixture(scope="module") def t_pm(): t_pm = Table( - [{'source': 'Fake 1', 'mu_ra': 113., 'mu_ra_err': 0.3, 'mu_dec': 113., 'mu_dec_err': 0.3, 'reference': 'Ref 1'}, - {'source': 'Fake 2', 'mu_ra': 145., 'mu_ra_err': 0.5, 'mu_dec': 113., 'mu_dec_err': 0.3, 'reference': 'Ref 1'}, - {'source': 'Fake 3', 'mu_ra': 55., 'mu_ra_err': 0.23, 'mu_dec': 113., 'mu_dec_err': 0.3, 'reference': 'Ref 2'}, - ]) + [ + { + "source": "Fake 1", + "mu_ra": 113.0, + "mu_ra_err": 0.3, + "mu_dec": 113.0, + "mu_dec_err": 0.3, + "reference": "Ref 1", + }, + { + "source": "Fake 2", + "mu_ra": 145.0, + "mu_ra_err": 0.5, + "mu_dec": 113.0, + "mu_dec_err": 0.3, + "reference": "Ref 1", + }, + { + "source": "Fake 3", + "mu_ra": 55.0, + "mu_ra_err": 0.23, + "mu_dec": 113.0, + "mu_dec_err": 0.3, + "reference": "Ref 2", + }, + ] + ) return t_pm def test_setup_db(db): # Some setup tasks to ensure some data exists in the database first - ref_data = [{'reference': 'Ref 1', 'doi': '10.1093/mnras/staa1522', 'bibcode': '2020MNRAS.496.1922B'}, - {'reference': 'Ref 2', 'doi': 'Doi2', 'bibcode': '2012yCat.2311....0C'}, - {'reference': 'Burn08', 'doi': 'Doi3', 'bibcode': '2008MNRAS.391..320B'}] - - source_data = [{'source': 'Fake 1', 'ra': 9.0673755, 'dec': 18.352889, 'reference': 'Ref 1'}, - {'source': 'Fake 2', 'ra': 9.0673755, 'dec': 18.352889, 'reference': 'Ref 1'}, - {'source': 'Fake 3', 'ra': 9.0673755, 'dec': 18.352889, 'reference': 'Ref 2'}, - ] + ref_data = [ + { + "reference": "Ref 1", + "doi": "10.1093/mnras/staa1522", + "bibcode": "2020MNRAS.496.1922B", + }, + {"reference": "Ref 2", "doi": "Doi2", "bibcode": "2012yCat.2311....0C"}, + {"reference": "Burn08", "doi": "Doi3", "bibcode": "2008MNRAS.391..320B"}, + ] + + source_data = [ + {"source": "Fake 1", "ra": 9.0673755, "dec": 18.352889, "reference": "Ref 1"}, + {"source": "Fake 2", "ra": 9.0673755, "dec": 18.352889, "reference": "Ref 1"}, + {"source": "Fake 3", "ra": 9.0673755, "dec": 18.352889, "reference": "Ref 2"}, + ] with db.engine.connect() as conn: conn.execute(db.Publications.insert().values(ref_data)) @@ -78,123 +124,276 @@ def test_setup_db(db): return db +@pytest.mark.filterwarnings( + "ignore::UserWarning" +) # suppress astroquery SIMBAD warnings def test_ingest_sources(db): # TODO: Test adding an alt name - source_data1 = Table([{'source': 'Fake 1', 'ra': 9.0673755, 'dec': 18.352889, 'reference': 'Ref 1'}, - {'source': 'Fake 6', 'ra': 10.0673755, 'dec': 18.352889, 'reference': 'Ref 2'}, - {'source': 'Fake 7', 'ra': 11.0673755, 'dec': 18.352889, 'reference': 'Ref 1'}]) - source_data5 = Table([{'source': 'Fake 5', 'ra': 9.06799, 'dec': 18.352889, 'reference': ''}]) - source_data8 = Table([{'source': 'Fake 8', 'ra': 9.06799, 'dec': 18.352889, 'reference': 'Ref 4'}]) - - ingest_sources(db, source_data1['source'], ras=source_data1['ra'], decs=source_data1['dec'], - references=source_data1['reference'], raise_error=True) - - ingest_sources(db, ['Barnard Star'], references='Ref 2', raise_error=True) - Barnard_star = db.query(db.Sources).filter(db.Sources.c.source == 'Barnard Star').astropy() + source_data1 = Table( + [ + { + "source": "Apple", + "ra": 10.0673755, + "dec": 17.352889, + "reference": "Ref 1", + }, + { + "source": "Orange", + "ra": 12.0673755, + "dec": -15.352889, + "reference": "Ref 2", + }, + { + "source": "Banana", + "ra": 119.0673755, + "dec": -28.352889, + "reference": "Ref 1", + }, + ] + ) + + ingest_sources( + db, + source_data1["source"], + ras=source_data1["ra"], + decs=source_data1["dec"], + references=source_data1["reference"], + raise_error=True, + ) + assert db.query(db.Sources).filter(db.Sources.c.source == "Apple").count() == 1 + assert db.query(db.Sources).filter(db.Sources.c.source == "Orange").count() == 1 + assert db.query(db.Sources).filter(db.Sources.c.source == "Banana").count() == 1 + + +@pytest.mark.filterwarnings( + "ignore::UserWarning" +) # suppress astroquery SIMBAD warnings +def test_ingest_source(db): + ingest_source(db, "Barnard Star", reference="Ref 2", raise_error=True) + + Barnard_star = ( + db.query(db.Sources).filter(db.Sources.c.source == "Barnard Star").astropy() + ) assert len(Barnard_star) == 1 - assert math.isclose(Barnard_star['ra'], 269.452, abs_tol=0.001) - assert math.isclose(Barnard_star['dec'], 4.6933, abs_tol=0.001) - - assert db.query(db.Sources).filter(db.Sources.c.source == 'Fake 1').count() == 1 - assert db.query(db.Sources).filter(db.Sources.c.source == 'Fake 6').count() == 1 - assert db.query(db.Sources).filter(db.Sources.c.source == 'Fake 7').count() == 1 - + assert math.isclose(Barnard_star["ra"][0], 269.452, abs_tol=0.001) + assert math.isclose(Barnard_star["dec"][0], 4.6933, abs_tol=0.001) + + source_data8 = { + "source": "Fake 8", + "ra": 9.06799, + "dec": 18.352889, + "reference": "Ref 4", + } with pytest.raises(SimpleError) as error_message: - ingest_sources(db, source_data8['source'], ras=source_data8['ra'], decs=source_data5['dec'], - references=source_data8['reference'], raise_error=True) - assert 'not in Publications table' in str(error_message.value) + ingest_source( + db, + source_data8["source"], + ra=source_data8["ra"], + dec=source_data8["dec"], + reference=source_data8["reference"], + raise_error=True, + ) + assert "not in Publications table" in str(error_message.value) + + source_data5 = { + "source": "Fake 5", + "ra": 9.06799, + "dec": 18.352889, + "reference": "", + } + with pytest.raises(SimpleError) as error_message: + ingest_source( + db, + source_data5["source"], + ra=source_data5["ra"], + dec=source_data5["dec"], + reference=source_data5["reference"], + raise_error=True, + ) + assert "blank" in str(error_message.value) with pytest.raises(SimpleError) as error_message: - ingest_sources(db, source_data5['source'], ras=source_data5['ra'], decs=source_data5['dec'], - references=source_data5['reference'], raise_error=True) - assert 'blank' in str(error_message.value) + ingest_source(db, "NotinSimbad", reference="Ref 1", raise_error=True) + assert "Coordinates are needed" in str(error_message.value) with pytest.raises(SimpleError) as error_message: - ingest_sources(db, ['NotinSimbad'], references='Ref 1', raise_error=True) - assert 'Coordinates are needed' in str(error_message.value) + ingest_source( + db, + "Fake 1", + ra=11.0673755, + dec=18.352889, + reference="Ref 1", + raise_error=True, + ) + assert "already exists" in str(error_message.value) def test_convert_spt_string_to_code(): # Test conversion of spectral types into numeric values - assert convert_spt_string_to_code(['M5.6']) == [65.6] - assert convert_spt_string_to_code(['T0.1']) == [80.1] - assert convert_spt_string_to_code(['Y2pec']) == [92] + assert convert_spt_string_to_code(["M5.6"]) == [65.6] + assert convert_spt_string_to_code(["T0.1"]) == [80.1] + assert convert_spt_string_to_code(["Y2pec"]) == [92] def test_ingest_parallaxes(db, t_plx): # Test ingest of parallax data - ingest_parallaxes(db, t_plx['source'], t_plx['plx'], t_plx['plx_err'], t_plx['plx_ref']) + ingest_parallaxes( + db, t_plx["source"], t_plx["plx"], t_plx["plx_err"], t_plx["plx_ref"] + ) - results = db.query(db.Parallaxes).filter(db.Parallaxes.c.reference == 'Ref 1').table() + results = ( + db.query(db.Parallaxes).filter(db.Parallaxes.c.reference == "Ref 1").table() + ) assert len(results) == 2 - results = db.query(db.Parallaxes).filter(db.Parallaxes.c.reference == 'Ref 2').table() + results = ( + db.query(db.Parallaxes).filter(db.Parallaxes.c.reference == "Ref 2").table() + ) assert len(results) == 1 - assert results['source'][0] == 'Fake 3' - assert results['parallax'][0] == 155 - assert results['parallax_error'][0] == 0.6 + assert results["source"][0] == "Fake 3" + assert results["parallax"][0] == 155 + assert results["parallax_error"][0] == 0.6 def test_ingest_proper_motions(db, t_pm): - ingest_proper_motions(db, t_pm['source'], t_pm['mu_ra'], t_pm['mu_ra_err'], - t_pm['mu_dec'], t_pm['mu_dec_err'], t_pm['reference']) - assert db.query(db.ProperMotions).filter(db.ProperMotions.c.reference == 'Ref 1').count() == 2 - results = db.query(db.ProperMotions).filter(db.ProperMotions.c.reference == 'Ref 2').table() + ingest_proper_motions( + db, + t_pm["source"], + t_pm["mu_ra"], + t_pm["mu_ra_err"], + t_pm["mu_dec"], + t_pm["mu_dec_err"], + t_pm["reference"], + ) + assert ( + db.query(db.ProperMotions) + .filter(db.ProperMotions.c.reference == "Ref 1") + .count() + == 2 + ) + results = ( + db.query(db.ProperMotions) + .filter(db.ProperMotions.c.reference == "Ref 2") + .table() + ) assert len(results) == 1 - assert results['source'][0] == 'Fake 3' - assert results['mu_ra'][0] == 55 - assert results['mu_ra_error'][0] == 0.23 + assert results["source"][0] == "Fake 3" + assert results["mu_ra"][0] == 55 + assert results["mu_ra_error"][0] == 0.23 def test_ingest_spectral_types(db): - data1 = Table([{'source': 'Fake 1', 'spectral_type': 'M5.6', 'regime': 'nir', 'reference': 'Ref 1'}, - {'source': 'Fake 2', 'spectral_type': 'T0.1', 'regime': 'nir', 'reference': 'Ref 1'}, - {'source': 'Fake 3', 'spectral_type': 'Y2pec', 'regime': 'nir', 'reference': 'Ref 2'}, - ]) - - data2 = Table([{'source': 'Fake 1', 'spectral_type': 'M5.6', 'reference': 'Ref 1'}, - {'source': 'Fake 2', 'spectral_type': 'T0.1', 'reference': 'Ref 1'}, - {'source': 'Fake 3', 'spectral_type': 'Y2pec', 'reference': 'Ref 2'}, - ]) - - data3 = Table([{'source': 'Fake 1', 'spectral_type': 'M5.6', 'regime': 'nir', 'reference': 'Ref 1'}, - {'source': 'Fake 2', 'spectral_type': 'T0.1', 'regime': 'nir', 'reference': 'Ref 1'}, - {'source': 'Fake 3', 'spectral_type': 'Y2pec', 'regime': 'nir', 'reference': 'Ref 4'}, - ]) - ingest_spectral_types(db, data1['source'], data1['spectral_type'], data1['reference'], data1['regime']) - assert db.query(db.SpectralTypes).filter(db.SpectralTypes.c.reference == 'Ref 1').count() == 2 - results = db.query(db.SpectralTypes).filter(db.SpectralTypes.c.reference == 'Ref 2').table() + data1 = Table( + [ + { + "source": "Fake 1", + "spectral_type": "M5.6", + "regime": "nir", + "reference": "Ref 1", + }, + { + "source": "Fake 2", + "spectral_type": "T0.1", + "regime": "nir", + "reference": "Ref 1", + }, + { + "source": "Fake 3", + "spectral_type": "Y2pec", + "regime": "nir", + "reference": "Ref 2", + }, + ] + ) + + # data2 = Table( + # [ + # {"source": "Fake 1", "spectral_type": "M5.6", "reference": "Ref 1"}, + # {"source": "Fake 2", "spectral_type": "T0.1", "reference": "Ref 1"}, + # {"source": "Fake 3", "spectral_type": "Y2pec", "reference": "Ref 2"}, + # ] + # ) + + data3 = Table( + [ + { + "source": "Fake 1", + "spectral_type": "M5.6", + "regime": "nir", + "reference": "Ref 1", + }, + { + "source": "Fake 2", + "spectral_type": "T0.1", + "regime": "nir", + "reference": "Ref 1", + }, + { + "source": "Fake 3", + "spectral_type": "Y2pec", + "regime": "nir", + "reference": "Ref 4", + }, + ] + ) + ingest_spectral_types( + db, data1["source"], data1["spectral_type"], data1["reference"], data1["regime"] + ) + assert ( + db.query(db.SpectralTypes) + .filter(db.SpectralTypes.c.reference == "Ref 1") + .count() + == 2 + ) + results = ( + db.query(db.SpectralTypes) + .filter(db.SpectralTypes.c.reference == "Ref 2") + .table() + ) assert len(results) == 1 - assert results['source'][0] == 'Fake 3' - assert results['spectral_type_string'][0] == 'Y2pec' - assert results['spectral_type_code'][0] == [92] + assert results["source"][0] == "Fake 3" + assert results["spectral_type_string"][0] == "Y2pec" + assert results["spectral_type_code"][0] == [92] # testing for publication error with pytest.raises(SimpleError) as error_message: - ingest_spectral_types(db, data3['source'], data3['spectral_type'], data3['regime'], data3['reference']) - assert 'The publication does not exist in the database' in str(error_message.value) + ingest_spectral_types( + db, + data3["source"], + data3["spectral_type"], + data3["regime"], + data3["reference"], + ) + assert "The publication does not exist in the database" in str( + error_message.value + ) def test_find_publication(db): assert not find_publication(db)[0] # False - assert find_publication(db, name='Ref 1')[0] # True - assert find_publication(db, name='Ref 1', doi='10.1093/mnras/staa1522')[0] # True - doi_search = find_publication(db, doi='10.1093/mnras/staa1522') + assert find_publication(db, name="Ref 1")[0] # True + assert find_publication(db, name="Ref 1", doi="10.1093/mnras/staa1522")[0] # True + doi_search = find_publication(db, doi="10.1093/mnras/staa1522") assert doi_search[0] # True - assert doi_search[1] == 'Ref 1' - bibcode_search = find_publication(db, bibcode='2020MNRAS.496.1922B') + assert doi_search[1] == "Ref 1" + bibcode_search = find_publication(db, bibcode="2020MNRAS.496.1922B") assert bibcode_search[0] # True - assert bibcode_search[1] == 'Ref 1' - multiple_matches = find_publication(db, name='Ref') + assert bibcode_search[1] == "Ref 1" + multiple_matches = find_publication(db, name="Ref") assert not multiple_matches[0] # False, multiple matches assert multiple_matches[1] == 2 # multiple matches - assert not find_publication(db, name='Ref 2', doi='10.1093/mnras/staa1522')[0] # False - assert not find_publication(db, name='Ref 2', bibcode='2020MNRAS.496.1922B')[0] # False - assert find_publication(db, name='Burningham_2008') == (1, 'Burn08') + assert not find_publication(db, name="Ref 2", doi="10.1093/mnras/staa1522")[ + 0 + ] # False + assert not find_publication(db, name="Ref 2", bibcode="2020MNRAS.496.1922B")[ + 0 + ] # False + assert find_publication(db, name="Burningham_2008") == (1, "Burn08") + def test_ingest_publication(db): # should fail if trying to add a duplicate record with pytest.raises(SimpleError) as error_message: - ingest_publication(db, publication='Ref 1', bibcode='2020MNRAS.496.1922B') - assert ' similar publication already exists' in str(error_message.value) + ingest_publication(db, publication="Ref 1", bibcode="2020MNRAS.496.1922B") + assert " similar publication already exists" in str(error_message.value) # TODO - Mock environment where ADS_TOKEN is not set. #117 @@ -202,85 +401,116 @@ def test_ingest_instrument(db): # TESTS WHICH SHOULD WORK # test adding just telescope - ingest_instrument(db, telescope='test') - telescope_db = db.query(db.Telescopes).filter(db.Telescopes.c.telescope == 'test').table() + ingest_instrument(db, telescope="test") + telescope_db = ( + db.query(db.Telescopes).filter(db.Telescopes.c.telescope == "test").table() + ) assert len(telescope_db) == 1 - assert telescope_db['telescope'][0] == 'test' + assert telescope_db["telescope"][0] == "test" # No longer supported just adding an instrument without a mode # test adding telescope and instrument # tel_test = 'test2' # inst_test = 'test3' # ingest_instrument(db, telescope=tel_test, instrument=inst_test) - # telescope_db = db.query(db.Telescopes).filter(db.Telescopes.c.telescope == tel_test).table() - # instrument_db = db.query(db.Instruments).filter(db.Instruments.c.instrument == inst_test).table() + # telescope_db = db.query(db.Telescopes). + # filter(db.Telescopes.c.telescope == tel_test).table() + # instrument_db = db.query(db.Instruments). + # filter(db.Instruments.c.instrument == inst_test).table() # assert len(telescope_db) == 1 # assert telescope_db['telescope'][0] == tel_test # assert len(instrument_db) == 1 # assert instrument_db['instrument'][0] == inst_test # test adding new telescope, instrument, and mode - tel_test = 'test4' - inst_test = 'test5' - mode_test = 'test6' + tel_test = "test4" + inst_test = "test5" + mode_test = "test6" ingest_instrument(db, telescope=tel_test, instrument=inst_test, mode=mode_test) - telescope_db = db.query(db.Telescopes).filter(db.Telescopes.c.telescope == tel_test).table() - instrument_db = db.query(db.Instruments).filter(and_(db.Instruments.c.mode == mode_test, - db.Instruments.c.instrument == inst_test, - db.Instruments.c.telescope == tel_test)).table() - assert len(telescope_db) == 1, 'Missing telescope insert' - assert telescope_db['telescope'][0] == tel_test + telescope_db = ( + db.query(db.Telescopes).filter(db.Telescopes.c.telescope == tel_test).table() + ) + instrument_db = ( + db.query(db.Instruments) + .filter( + and_( + db.Instruments.c.mode == mode_test, + db.Instruments.c.instrument == inst_test, + db.Instruments.c.telescope == tel_test, + ) + ) + .table() + ) + assert len(telescope_db) == 1, "Missing telescope insert" + assert telescope_db["telescope"][0] == tel_test assert len(instrument_db) == 1 - assert instrument_db['instrument'][0] == inst_test - assert instrument_db['mode'][0] == mode_test + assert instrument_db["instrument"][0] == inst_test + assert instrument_db["mode"][0] == mode_test # test adding common mode name for new telescope, instrument - tel_test = 'test4' - inst_test = 'test5' - mode_test = 'Prism' + tel_test = "test4" + inst_test = "test5" + mode_test = "Prism" print(db.query(db.Telescopes).table()) print(db.query(db.Instruments).table()) ingest_instrument(db, telescope=tel_test, instrument=inst_test, mode=mode_test) - mode_db = db.query(db.Instruments).filter(and_(db.Instruments.c.mode == mode_test, - db.Instruments.c.instrument == inst_test, - db.Instruments.c.telescope == tel_test)).table() + mode_db = ( + db.query(db.Instruments) + .filter( + and_( + db.Instruments.c.mode == mode_test, + db.Instruments.c.instrument == inst_test, + db.Instruments.c.telescope == tel_test, + ) + ) + .table() + ) assert len(mode_db) == 1 - assert mode_db['mode'][0] == mode_test + assert mode_db["mode"][0] == mode_test # TESTS WHICH SHOULD FAIL # test with no variables provided with pytest.raises(SimpleError) as error_message: ingest_instrument(db) - assert 'Telescope, Instrument, and Mode must be provided' in str(error_message.value) + assert "Telescope, Instrument, and Mode must be provided" in str( + error_message.value + ) # test with mode but no instrument or telescope with pytest.raises(SimpleError) as error_message: - ingest_instrument(db, mode='test') - assert 'Telescope, Instrument, and Mode must be provided' in str(error_message.value) + ingest_instrument(db, mode="test") + assert "Telescope, Instrument, and Mode must be provided" in str( + error_message.value + ) + # TODO: test for ingest_photometry # TODO: test for ingest_spectra + def test_companion_relationships(db): -# testing companion ingest -## trying no companion + # testing companion ingest + # trying no companion with pytest.raises(SimpleError) as error_message: - ingest_companion_relationships(db, 'Fake 1', None, "Sibling" ) + ingest_companion_relationships(db, "Fake 1", None, "Sibling") assert "Make sure all require parameters are provided." in str(error_message.value) - ## trying companion == source + # trying companion == source with pytest.raises(SimpleError) as error_message: - ingest_companion_relationships(db, 'Fake 1', 'Fake 1', "Sibling" ) - assert "Source cannot be the same as companion name" in str(error_message.value) + ingest_companion_relationships(db, "Fake 1", "Fake 1", "Sibling") + assert "Source cannot be the same as companion name" in str(error_message.value) - ## trying negative separation + # trying negative separation with pytest.raises(SimpleError) as error_message: - ingest_companion_relationships(db, 'Fake 1', 'Bad Companion', "Sibling", projected_separation_arcsec = -5) - assert 'cannot be negative' in str(error_message.value) + ingest_companion_relationships( + db, "Fake 1", "Bad Companion", "Sibling", projected_separation_arcsec=-5 + ) + assert "cannot be negative" in str(error_message.value) - ## trying negative separation error + # trying negative separation error with pytest.raises(SimpleError) as error_message: - ingest_companion_relationships(db, 'Fake 1', 'Bad Companion', "Sibling" , projected_separation_error = -5) - assert 'cannot be negative' in str(error_message.value) - + ingest_companion_relationships( + db, "Fake 1", "Bad Companion", "Sibling", projected_separation_error=-5 + ) + assert "cannot be negative" in str(error_message.value)