Skip to content

Commit

Permalink
Simple fix for handling slash in column names
Browse files Browse the repository at this point in the history
  • Loading branch information
JulienPeloton committed Jan 14, 2022
1 parent 36ab0bb commit e8fbc01
Show file tree
Hide file tree
Showing 3 changed files with 44 additions and 1 deletion.
36 changes: 35 additions & 1 deletion src/main/scala/com/astrolabsoftware/sparkfits/FitsLib.scala
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,15 @@ object FitsLib {

/**
* Decompose each line of the header into (key, value).
* Note the white space before the slash to include colnames with slash:
* a=toto/tutu / a comment
* TTYPEn would be `toto/tutu`
*
* I reckon This is not very robust though as rows such as
* a=toto /tutu / a comment
* where the value of TTYPEn is expected to be `toto / tutu` would not
* be considered, and it would return only `toto`.
*
*
* @param header : (Array[String])
* The header of the HDU.
Expand All @@ -101,10 +110,35 @@ object FitsLib {
def parseHeader(header : Array[String]) : Map[String, String] = {
header.map(x => x.split("="))
.filter(x => x.size > 1)
.map(x => (x(0).trim(), x(1).split("/")(0).trim()))
.map(x => (x(0).trim(), x(1).split(" /")(0).trim()))
.toMap
}

/**
* This is a better code than above, but it does not work if
* there is no comment section on the card...
* a=toto/tutu / a comment --> OK
* a=toto/tutu --> NOT OK
* Unfortunately, both exist :-(
*
* @param header : (Array[String])
* The header of the HDU.
* @return (Map[String, String]), map array with (keys, values).
*
*/
// def parseHeader(header : Array[String]) : Map[String, String] = {
// header.map(x => x.split("="))
// .filter(x => x.size > 1)
// .map{x =>
// val head = x(0).trim()
// val ttype_ = x(1).split("/").map(y => y.trim())
//
// val ttype = ttype_.slice(0, ttype_.size - 1).mkString("/")
//
// (head, ttype)
// }.toMap
// }

/**
*
* Remove single quotes around a string, and trim the resulting string.
Expand Down
Binary file added src/test/resources/colnames_with_slash.fits
Binary file not shown.
Original file line number Diff line number Diff line change
Expand Up @@ -251,4 +251,13 @@ class packageTest extends FunSuite with BeforeAndAfterAll {
.load(fn)
assert(results.select(col("target")).first.getString(0) == "NGC0000000")
}

// Test slash in columns
test("Slash in names: can you read column names with slash?") {
val fn = "src/test/resources/colnames_with_slash.fits"
val results = spark.read.format("com.astrolabsoftware.sparkfits")
.option("hdu", 1)
.load(fn)
assert(results.columns.deep == Array("lsst/u_MEAN", "lsst/g_MEAN", "euclid/VIS_MEAN", "euclid/Y_MEAN").deep)
}
}

0 comments on commit e8fbc01

Please sign in to comment.