initial

Author: qnt-dmitry

.gitignore

@@ -1,2 +1,4 @@
 *.class
 *.log
+target/
+.idea/

build.sbt

@@ -0,0 +1,15 @@
+name := "qnt"
+
+version := "0.1"
+
+scalaVersion := "2.11.12"
+
+resolvers += "Unidata" at "https://artifacts.unidata.ucar.edu/repository/unidata-all"
+libraryDependencies += "edu.ucar" % "cdm-core" % "5.2.0"
+libraryDependencies += "edu.ucar" % "netcdf" % "4.3.22"
+
+resolvers += "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots"
+libraryDependencies += "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.10.1"
+
+resolvers += "Sonatype Releases" at "http://oss.sonatype.org/content/repositories/releases"
+libraryDependencies += "org.scala-saddle" %% "saddle-core" % "1.3.+"

project/build.properties

@@ -0,0 +1 @@
+sbt.version = 1.3.3

src/main/scala/qnt/data.scala

@@ -0,0 +1,137 @@
+package qnt
+
+import java.net.{HttpURLConnection, URL}
+import java.time.LocalDate
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper}
+import org.saddle.Frame
+import org.slf4j.LoggerFactory
+import ucar.ma2.ArrayDouble
+import ucar.nc2.NetcdfFile
+
+import scala.collection.immutable.{Map => ImmutableMap}
+import scala.util.control.NonFatal
+
+case class StockInfo (
+                  id: String,
+                  cik: Option[String],
+                  exchange: String,
+                  symbol: String,
+                  name: String,
+                  sector: Option[String],
+                  props: ImmutableMap[String, Any]
+                ) {
+
+  def this(props: Map[String, Any]) = this(
+    props("id").asInstanceOf[String],
+    //props("FIGI").asInstanceOf[String],
+    props.get("cik") match {
+      case Some(s) => {
+        val str = s.asInstanceOf[String]
+        if (str != null && str.length > 0) Some(str) else None
+      }
+      case _ => None
+    },
+    props("exchange").asInstanceOf[String],
+    props("symbol").asInstanceOf[String],
+    props("name").asInstanceOf[String],
+    {
+      val s = props("sector").asInstanceOf[String]
+      if (s == null || s.length < 1) None else Some(s)
+    },
+    props - "id" - "cik" - "exchange" - "symbol" - "name" - "sector"
+  )
+
+}
+
+object data {
+
+  def loadStockList(
+                     minDate: LocalDate = LocalDate.of(2007, 1, 1),
+                     maxDate: LocalDate = LocalDate.now()
+                   ) : List[StockInfo] = {
+
+    val uri = s"assets?min_date=$minDate&max_date=$maxDate"
+    val dataBytes = loadWithRetry(uri)
+    if(dataBytes == null) {
+      return List.empty
+    }
+    val lst = OBJECT_MAPPER.readValue[List[Map[String, Any]]](dataBytes)
+    lst.map(p => new StockInfo(p))
+  }
+
+  def loadStockDailySeries(
+                            ids: List[String],
+                            minDate: LocalDate = LocalDate.of(2007, 1, 1),
+                            maxDate: LocalDate = LocalDate.now()
+                          ) : Frame[LocalDate, String, Float] = {
+    var uri = "data"
+    var params = Map(
+      "assets" -> ids,
+      "min_date" -> minDate.toString,
+      "max_date" -> maxDate.toString
+    )
+    val  ad: ArrayDouble = null;
+
+    var dataBytes = loadWithRetry(uri, params)
+    val dataNetcdf = NetcdfFile.openInMemory("data", dataBytes)
+    null
+  }
+
+  private val LOG = LoggerFactory.getLogger(getClass)
+  private val RETRIES = 5
+  private val TIMEOUT = 60*1000
+  private val OBJECT_MAPPER = new ObjectMapper() with ScalaObjectMapper
+
+  OBJECT_MAPPER.registerModule(DefaultScalaModule)
+
+  def loadIndexList() = ???
+
+  def loadIndexSeries() = ???
+
+  private def loadWithRetry(uri: String, dataObj: Any = null): Array[Byte] = {
+    val urlStr = baseUrl + uri
+    val url = new URL(urlStr)
+    for (t <- 1 to RETRIES) {
+      var exc: Exception = null
+      var conn: HttpURLConnection = null
+      try {
+        conn = url.openConnection().asInstanceOf[HttpURLConnection]
+        conn.setConnectTimeout(TIMEOUT)
+        conn.setReadTimeout(TIMEOUT)
+        conn.setDoOutput(dataObj != null)
+        conn.setRequestMethod(if(dataObj == null) "GET" else "POST")
+        conn.setUseCaches(false)
+        conn.setDoInput(true);
+        if(dataObj != null) {
+          val dataBytes = OBJECT_MAPPER.writeValueAsBytes(dataObj)
+          // conn.setRequestProperty("Content-Type", "application/json; utf-8")
+          conn.setRequestProperty("Content-Length", Integer.toString(dataBytes.length))
+          conn.getOutputStream.write(dataBytes)
+          conn.getOutputStream.flush()
+        }
+        val code = conn.getResponseCode
+        if(code == 404) {
+          return null
+        }
+        if(code != 200) {
+          throw new IllegalStateException(s"Wrong status code: $code")
+        }
+        val is = conn.getInputStream
+        return is.readAllBytes()
+      } catch {
+        case NonFatal(e) => LOG.warn(s"Download exception, URL: $urlStr", e)
+      } finally {
+        if (conn != null) conn.disconnect()
+      }
+    }
+    throw new IllegalStateException(s"Data was not loaded, URL: $urlStr")
+  }
+
+  private def baseUrl = {
+    System.getenv().getOrDefault("DATA_BASE_URL", "http://127.0.0.1:8000/")
+  }
+
+}
+

src/main/scala/qnt/rn.scala

@@ -0,0 +1,36 @@
+package qnt
+
+
+import java.time.LocalDate
+
+import qnt.{data => qndata}
+
+
+object Obj extends App {
+    val fruits = List("apple", "banana", "lime", "orange")
+
+    val fruitLengths = for {
+      f <- fruits
+      if f.length > 4
+    } yield f.length
+
+    Console.println(fruitLengths)
+
+  val a = List(1,2,3,4)
+
+
+  var s = a.reduce((a1,a2) => a1*a2)
+  println(s)
+
+
+  val lst = qndata.loadStockList(
+    LocalDate.of(2007, 1, 1),
+    LocalDate.of(2019, 7, 7)
+  )
+
+  val ids = lst.take(20).map(i => i.id)
+
+  val azaza = qndata.loadStockDailySeries(ids)
+
+  lst.filter(e => e.sector.isEmpty).foreach(println)
+}