Skip to content

Update to Stock Quote History v8 #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Update to Stock Quote History v8 #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9c570a1
Update to version 8 of Yahoo Finance Stock Quote History API.
manoelcampos Dec 6, 2019
4d0d9d8
Updates versions.
manoelcampos Dec 6, 2019
8726f91
When multiple stocks are provided and the query fails for one
manoelcampos Dec 7, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.zavtech</groupId>
<artifactId>morpheus-yahoo</artifactId>
<version>0.9.21</version>
<version>0.10.0</version>
<packaging>jar</packaging>

<name>Morpheus-Yahoo</name>
Expand Down Expand Up @@ -79,7 +79,7 @@
<dependency>
<groupId>com.zavtech</groupId>
<artifactId>morpheus-viz</artifactId>
<version>0.9.16</version>
<version>0.9.21</version>
<scope>test</scope>
</dependency>

Expand Down
137 changes: 137 additions & 0 deletions src/main/java/com/zavtech/morpheus/yahoo/YahooIndicatorsJsonParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
package com.zavtech.morpheus.yahoo;

import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;

import java.io.InputStream;
import java.io.InputStreamReader;
import java.time.LocalDate;
import java.time.LocalDateTime;

/**
* Parses the JSON returned by the Yahoo Finance Quote service,
* enabling reading stocks metadata and indicators such as
* open, high, low and close value (between other data).
*
* @author Manoel Campos da Silva Filho
*/
public class YahooIndicatorsJsonParser {
/**
* The name of the fields representing the indicators in the Yahoo Finance Quote service response.
*/
public enum Indicator {OPEN, HIGH, LOW, CLOSE, VOLUME, ADJCLOSE};

/**
* A timestamp array where each item correspond to a given date
* represented in seconds since 1970, jan 1st.
*/
private final JsonArray timestamps;

/**
* Values for the {@link Indicator}s, except the {@link Indicator#ADJCLOSE}.
* Each indicator is a property in the JSON object.
* Each property is an array containing the double value
* for the indicator (one for each date defined in the {@link #timestamps} array).
*
* @see #adjClose
*/
private final JsonObject quotes;

/**
* Values for the {@link Indicator#ADJCLOSE}.
* It is an array containing for the adjusted close values
* (one for each date defined in the {@link #timestamps} array).
*/
private final JsonArray adjClose;

/**
* Instantiates the class, parsing the JSON response from a request sent to the Yahoo Finance Quote service.
* It uses a given input stream to obtain the response data.
* @param stream the input stream to read the JSON response data
*/
public YahooIndicatorsJsonParser(final InputStream stream){
final JsonObject result = parseYahooFinanceResult(new InputStreamReader(stream));
final JsonObject indicators = result.getAsJsonObject("indicators");

timestamps = result.getAsJsonArray("timestamp");
quotes = indicators.getAsJsonArray("quote").get(0).getAsJsonObject();
adjClose = indicators.getAsJsonArray("adjclose").get(0).getAsJsonObject().getAsJsonArray("adjclose");
}

/**
* Parses a JSON response got from a reader and try to return the JSON object containing
* the stocks quotes.
*
* @param reader the reader to get the JSON String from
* @return an {@link JsonObject} containing the data for the chart.result JSON field
* or an empty object if the result is empty
*/
private JsonObject parseYahooFinanceResult(final InputStreamReader reader) {
final JsonElement element = new JsonParser().parse(reader);
if(!element.isJsonObject()){
throw new IllegalStateException("The Yahoo Finance response is not a JSON object as expected.");
}

try {
return element
.getAsJsonObject()
.getAsJsonObject("chart")
.getAsJsonArray("result")
.get(0)
.getAsJsonObject();
}catch(ArrayIndexOutOfBoundsException|NullPointerException e){
return new JsonObject();
}
}

/**
* Gets the quote timestamp at a given position of the timestamp array
* and converts to a LocalDate value.
* @param index the desired position in the array
* @return the quote date
*/
public LocalDate getDate(final int index){
return secondsToLocalDate(timestamps.get(index).getAsLong());
}

/**
* Converts a given number of seconds (timestamp) since 1970/jan/01 to LocalDate.
* This timestamp value is the date format in Yahoo Finance (at least since v8).
* @param seconds the number of seconds to convert
* @return a LocalDate representing that number of seconds
*/
public LocalDate secondsToLocalDate(final long seconds) {
return LocalDateTime.of(1970, 1, 1, 0, 0).plusSeconds(seconds).toLocalDate();
}

/**
* Gets the value for a specific metric of the stock in a given date,
* represented by the index of the quotes array.
* The metric values are
* @param index the desired position in the array
* @return the metric value.
*/
public double getQuote(final Indicator indicator, final int index){
if(indicator.equals(Indicator.ADJCLOSE)) {
return getJsonDoubleValue(adjClose.get(index));
}

final String metricName = indicator.name().toLowerCase();
final JsonElement element = quotes.getAsJsonArray(metricName).get(index);
return getJsonDoubleValue(element);
}

private double getJsonDoubleValue(final JsonElement element){
return element.isJsonNull() ? Double.NaN : element.getAsDouble();
}

public boolean isEmpty() {
return timestamps.size() == 0;
}

public int rows(){
return timestamps.size();
}
}
57 changes: 21 additions & 36 deletions src/main/java/com/zavtech/morpheus/yahoo/YahooQuoteHistorySource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
package com.zavtech.morpheus.yahoo;

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.time.Duration;
import java.time.LocalDate;
Expand All @@ -40,11 +39,12 @@
import com.zavtech.morpheus.range.Range;
import com.zavtech.morpheus.util.Asserts;
import com.zavtech.morpheus.util.IO;
import com.zavtech.morpheus.util.TextStreamReader;
import com.zavtech.morpheus.util.http.HttpClient;
import com.zavtech.morpheus.util.http.HttpException;
import com.zavtech.morpheus.util.http.HttpHeader;

import static com.zavtech.morpheus.yahoo.YahooIndicatorsJsonParser.Indicator;

/**
* A DataFrameSource implementation that loads historical quote data from Yahoo Finance using their CSV API.
*
Expand All @@ -58,7 +58,7 @@ public class YahooQuoteHistorySource extends DataFrameSource<LocalDate,YahooFiel

private static final String CRUMB_URL = "https://query1.finance.yahoo.com/v1/test/getcrumb";
private static final String COOKIE_URL = "https://finance.yahoo.com/quote/SPY?p=SPY";
private static final String QUOTE_URL = "https://query1.finance.yahoo.com/v7/finance/download/%s?period1=%d&period2=%d&interval=1d&events=history&crumb=%s";
private static final String QUOTE_URL = "https://query1.finance.yahoo.com/v8/finance/chart/%s?period1=%d&period2=%d&interval=1d&events=history&crumb=%s";

private static Predicate<LocalDate> weekdayPredicate = date -> {
if (date == null) {
Expand Down Expand Up @@ -125,23 +125,20 @@ public DataFrame<LocalDate,YahooField> read(Consumer<Options> configurator) thro
final int code = response.getStatus().getCode();
throw new HttpException(httpRequest, "Yahoo Finance responded with status code " + code, null);
} else {
final InputStream stream = response.getStream();
final TextStreamReader reader = new TextStreamReader(stream);
if (reader.hasNext()) reader.nextLine(); //Swallow the header
final YahooIndicatorsJsonParser indicators = new YahooIndicatorsJsonParser(response.getStream());
final Index<LocalDate> rowKeys = createDateIndex(options);
final Index<YahooField> colKeys = Index.of(fields.copy());
final DataFrame<LocalDate,YahooField> frame = DataFrame.ofDoubles(rowKeys, colKeys);
final DataFrameCursor<LocalDate,YahooField> cursor = frame.cursor();
while (reader.hasNext()) {
final String line = reader.nextLine();
final String[] elements = line.split(",");
final LocalDate date = parseDate(elements[0]);
final double open = Double.parseDouble(elements[1]);
final double high = Double.parseDouble(elements[2]);
final double low = Double.parseDouble(elements[3]);
final double close = Double.parseDouble(elements[4]);
final double closeAdj = Double.parseDouble(elements[5]);
final double volume = Double.parseDouble(elements[6]);
for (int i = 0; i < indicators.rows(); i++) {
final LocalDate date = indicators.getDate(i);
final double open = indicators.getQuote(Indicator.OPEN, i);
final double high = indicators.getQuote(Indicator.HIGH, i);
final double low = indicators.getQuote(Indicator.LOW, i);
final double close = indicators.getQuote(Indicator.CLOSE, i);
final double closeAdj = indicators.getQuote(Indicator.ADJCLOSE, i);

final double volume = indicators.getQuote(Indicator.VOLUME, i);
final double splitRatio = Math.abs(closeAdj - close) > 0.00001d ? closeAdj / close : 1d;
final double adjustment = options.dividendAdjusted ? splitRatio : 1d;
if (options.paddedHolidays) {
Expand Down Expand Up @@ -169,6 +166,7 @@ public DataFrame<LocalDate,YahooField> read(Consumer<Options> configurator) thro
if (options.paddedHolidays) {
frame.fill().down(2);
}

calculateChanges(frame);
return Optional.of(frame);
}
Expand All @@ -181,7 +179,6 @@ public DataFrame<LocalDate,YahooField> read(Consumer<Options> configurator) thro
}
}


/**
* Returns the date index to initialize the row axis
* @param options the options for the request
Expand Down Expand Up @@ -233,23 +230,6 @@ private URL createURL(String symbol, LocalDate start, LocalDate end) throws Exce
}
}

/**
* Parses dates in the formatSqlDate YYYY-MM-DD
* @param dateString the string to parse
* @return the parsed date value
*/
private LocalDate parseDate(String dateString) {
if (dateString == null) {
return null;
} else {
final String[] elements = dateString.trim().split("-");
final int year = Integer.parseInt(elements[0]);
final int month = Integer.parseInt(elements[1]);
final int date = Integer.parseInt(elements[2]);
return LocalDate.of(year, month, date);
}
}


/**
* Returns the cookies to send with the request
Expand Down Expand Up @@ -403,11 +383,16 @@ public Options withDividendAdjusted(boolean dividendAdjusted) {
}
}



public static void main(String[] args) {
final LocalDate start = LocalDate.of(2010, 1, 1);
final LocalDate end = LocalDate.of(2012, 1, 1);
final String brazilianStock = "MGLU3.sa";
System.out.printf("%n%s quotes from %s to %s%n", brazilianStock, start, end);
final YahooFinance yahoo = new YahooFinance();
final DataFrame<LocalDate, String> returns = yahoo.getDailyReturns(start, end, Array.of(brazilianStock, "BID3.sa", "ITUB4.sa"));
returns.out().print(returns.rowCount());
System.out.println();

final Array<String> tickers = Array.of("AAPL", "MSFT", "ORCL", "GE", "C");
final YahooQuoteHistorySource source = new YahooQuoteHistorySource();
tickers.forEach(ticker -> {
Expand Down
16 changes: 10 additions & 6 deletions src/main/java/com/zavtech/morpheus/yahoo/YahooReturnSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,7 @@
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.*;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.Stream;
Expand Down Expand Up @@ -76,9 +73,9 @@ public DataFrame<LocalDate,String> read(Consumer<Options> configurator) throws D
final Options options = initOptions(new Options(), configurator);
final List<Callable<DataFrame<LocalDate,String>>> tasks = createTasks(options);
final List<Future<DataFrame<LocalDate,String>>> futures = executor.invokeAll(tasks);
final List<DataFrame<LocalDate,String>> frames = futures.stream().map(Try::get).collect(Collectors.toList());
final List<DataFrame<LocalDate,String>> frames = futures.stream().map(this::futureGet).filter(f -> !f.isEmpty()).collect(Collectors.toList());
final DataFrame<LocalDate,String> result = DataFrame.combineFirst(frames);
final DataFrame<LocalDate,String> returns = result.cols().select(options.tickers).rows().sort(true).copy();
final DataFrame<LocalDate,String> returns = result.cols().select(col -> options.tickers.contains(col.key())).rows().sort(true).copy();
if (options.emaHalfLife == null) {
return returns;
} else {
Expand All @@ -91,6 +88,13 @@ public DataFrame<LocalDate,String> read(Consumer<Options> configurator) throws D
}
}

private DataFrame<LocalDate, String> futureGet(final Future<DataFrame<LocalDate, String>> future) {
try {
return future.get();
} catch (Exception e) {
return DataFrame.empty();
}
}

/**
* Returns the list of tasks for the request specified
Expand Down