From 7b2bff876de25ee02de4d63e2ba04c58be4b8f1f Mon Sep 17 00:00:00 2001 From: Krzysztof Naglik Date: Sat, 15 Jul 2023 16:17:29 +0200 Subject: [PATCH] Improve URL serialization (#3) --- .github/workflows/ci.yaml | 4 +- api/url.go | 102 ----------- api/url_test.go | 46 ----- api/flight.go => flight.go | 23 ++- api/flight_test.go => flight_test.go | 2 +- api/flight_v2.go => flight_v2.go | 6 +- api/flight_v2_test.go => flight_v2_test.go | 2 +- main.go | 3 +- api/serialize_city.go => serialize_city.go | 16 +- url.go | 201 +++++++++++++++++++++ url_test.go | 112 ++++++++++++ 11 files changed, 353 insertions(+), 164 deletions(-) delete mode 100644 api/url.go delete mode 100644 api/url_test.go rename api/flight.go => flight.go (95%) rename api/flight_test.go => flight_test.go (98%) rename api/flight_v2.go => flight_v2.go (99%) rename api/flight_v2_test.go => flight_v2_test.go (98%) rename api/serialize_city.go => serialize_city.go (93%) create mode 100644 url.go create mode 100644 url_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 689c1e0..f8a8a7b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,7 +8,7 @@ on: - '**' jobs: - integration_test: + unit_test: name: Run tests runs-on: ubuntu-22.04 steps: @@ -21,6 +21,6 @@ jobs: - name: Check out repository code uses: actions/checkout@v3 - - name: Run integration test + - name: Run unit tests run: | go test ./... -v diff --git a/api/url.go b/api/url.go deleted file mode 100644 index 4530b64..0000000 --- a/api/url.go +++ /dev/null @@ -1,102 +0,0 @@ -package api - -import ( - "time" -) - -func createURL(a []byte) string { - var chars = [...]byte{ - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', - 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', - 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_'} - - var c []byte - var d byte // or chars[64] - e, f := 0, 0 - for ; e < len(a)-2; e += 3 { - h := a[e] - k := a[e+1] - l := a[e+2] - m := chars[h>>2] - h = chars[(h&3)<<4|k>>4] - k = chars[(k&15)<<2|l>>6] - l = chars[l&63] - c = append(c, m) - c = append(c, h) - c = append(c, k) - c = append(c, l) - f += 1 - } - - m := 0 - l := d - - switch len(a) - e { - case 2: - m := a[e+1] - element_id := int((m & 15) << 2) - if len(chars) > element_id { - l = chars[element_id] - } else { - l = d - } - case 1: - a_prim := int(a[e]) - c = append(c, chars[a_prim>>2]) - c = append(c, chars[(a_prim&3)<<4|m>>4]) - c = append(c, l) - c = append(c, d) - } - - return string(c[:]) -} - -func subSerialize( - departureDate time.Time, - returnDate time.Time, - serializedDepartureCity string, - serializedArrivalCity string, -) []byte { - subSerialize := []byte{ - 8, 28, 16, 2, 26, 40, 18, 10, - 50, 48, 50, 51, 45, 48, 55, 45, 48, 51, // departure date - 106, 12, 8, 2, 18, 8, // ?? - 47, 109, 47, 48, 56, 52, 53, 98, 114, 12, 8, 3, 18, 8, // departure - 47, 109, 47, 48, 53, 121, 119, 103, 26, 40, 18, 10, // arrival - 50, 48, 50, 51, 45, 48, 55, 45, 48, 55, // return date - 106, 12, 8, 3, 18, 8, // ?? - 47, 109, 47, 48, 53, 121, 119, 103, 114, 12, 8, 2, 18, 8, // departure - 47, 109, 47, 48, 56, 52, 53, 98, 64, 1, 72, 1, 112, 1, 130, // arrival - 1, 11, 8, 255, 255, 255, 255, 255, 255, 255, 255, 255, 1, 152, 1, 1, // ?? - } - // fmt.Println(departureDate.Format("2006-01-02")) - copy(subSerialize[8:], []byte(departureDate.Format("2006-01-02"))) - copy(subSerialize[50:], []byte(returnDate.Format("2006-01-02"))) - copy(subSerialize[24:], []byte(serializedDepartureCity)) - copy(subSerialize[38:], []byte(serializedArrivalCity)) - copy(subSerialize[66:], []byte(serializedArrivalCity)) - copy(subSerialize[80:], []byte(serializedDepartureCity)) - return subSerialize -} - -func CreateFullURL( - departureDate time.Time, - returnDate time.Time, - departureCity string, - arivalCity string, -) (string, error) { - serializedDepartureCity, err := GetSerializedCityName(departureCity) - if err != nil { - return "", err - } - serializedArrivalCity, err := GetSerializedCityName(arivalCity) - if err != nil { - return "", err - } - - url := "https://www.google.com/travel/flights/search?tfs=" + - createURL(subSerialize(departureDate, returnDate, serializedDepartureCity, serializedArrivalCity)) - - return url, nil -} diff --git a/api/url_test.go b/api/url_test.go deleted file mode 100644 index 3550122..0000000 --- a/api/url_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package api - -import ( - "testing" - "time" -) - -func TestCreateFullURL1(t *testing.T) { - expectedUrl := "https://www.google.com/travel/flights/search?tfs=CBwQAhooEgoyMDIzLTA3LTExagwIAhIIL20vMDg0NWJyDAgDEggvbS8wNTZfeRooEgoyMDIzLTA3LTE3agwIAxIIL20vMDU2X3lyDAgCEggvbS8wODQ1YkABSAFwAYIBCwj___________8BmAEB" - - departureDate, err := time.Parse("2006-01-02", "2023-07-11") - if err != nil { - t.Fatalf("Error while creating departure date") - } - returnDate, err := time.Parse("2006-01-02", "2023-07-17") - if err != nil { - t.Fatalf("Error while creating return date") - } - url, err := CreateFullURL(departureDate, returnDate, "Wrocław", "Madryt") - if err != nil { - t.Fatalf(err.Error()) - } - if url != expectedUrl { - t.Fatalf("Created url is different than expected. Created: %s Expected: %s", url, expectedUrl) - } -} - -func TestCreateFullURL2(t *testing.T) { - expectedUrl := "https://www.google.com/travel/flights/search?tfs=CBwQAhooEgoyMDIzLTA4LTIwagwIAhIIL20vMDV5d2dyDAgDEggvbS8wMTU2cRooEgoyMDIzLTA5LTAxagwIAxIIL20vMDE1NnFyDAgCEggvbS8wNXl3Z0ABSAFwAYIBCwj___________8BmAEB" - - departureDate, err := time.Parse("2006-01-02", "2023-08-20") - if err != nil { - t.Fatalf("Error while creating departure date") - } - returnDate, err := time.Parse("2006-01-02", "2023-09-01") - if err != nil { - t.Fatalf("Error while creating return date") - } - url, err := CreateFullURL(departureDate, returnDate, "Praga", "Berlin") - if err != nil { - t.Fatalf(err.Error()) - } - if url != expectedUrl { - t.Fatalf("Created url is different than expected. Created: %s Expected: %s", url, expectedUrl) - } -} diff --git a/api/flight.go b/flight.go similarity index 95% rename from api/flight.go rename to flight.go index 4df8f85..f54ba1a 100644 --- a/api/flight.go +++ b/flight.go @@ -1,4 +1,4 @@ -package api +package main import ( "bufio" @@ -107,17 +107,30 @@ func getPriceSuffix(unit currency.Unit) string { } func GetFlights( - departureDate time.Time, + date time.Time, returnDate time.Time, - departureCity string, - arivalCity string, + srcCity string, + dstCity string, unit currency.Unit, ) ([]flight, error) { if !isSupportedCurrency(unit) { return nil, fmt.Errorf(unit.String() + " is not supproted yet") } - url, err := CreateFullURL(departureDate, returnDate, departureCity, arivalCity) + url, err := SerializeUrl( + date, + returnDate, + []string{srcCity}, + []string{}, + []string{dstCity}, + []string{}, + 1, + unit, + AnyStops, + Economy, + RoundTrip, + ) + if err != nil { return nil, err } diff --git a/api/flight_test.go b/flight_test.go similarity index 98% rename from api/flight_test.go rename to flight_test.go index a00c69b..a5499ff 100644 --- a/api/flight_test.go +++ b/flight_test.go @@ -1,4 +1,4 @@ -package api +package main import ( "testing" diff --git a/api/flight_v2.go b/flight_v2.go similarity index 99% rename from api/flight_v2.go rename to flight_v2.go index a6e3261..0f6bba9 100644 --- a/api/flight_v2.go +++ b/flight_v2.go @@ -1,4 +1,4 @@ -package api +package main import ( "bufio" @@ -66,11 +66,11 @@ func (t offer) String() string { } func GetRawData(date, returnDate time.Time, originCity, targetCity string) string { - serializedOriginCity, err := GetSerializedCityName(originCity) + serializedOriginCity, err := AbbrCity(originCity) if err != nil { log.Fatalf(err.Error()) } - serializedTargetCity, err := GetSerializedCityName(targetCity) + serializedTargetCity, err := AbbrCity(targetCity) if err != nil { log.Fatalf(err.Error()) } diff --git a/api/flight_v2_test.go b/flight_v2_test.go similarity index 98% rename from api/flight_v2_test.go rename to flight_v2_test.go index e333373..15cdf2c 100644 --- a/api/flight_v2_test.go +++ b/flight_v2_test.go @@ -1,4 +1,4 @@ -package api +package main import ( "testing" diff --git a/main.go b/main.go index 8e31f4e..bf0b5b7 100644 --- a/main.go +++ b/main.go @@ -5,7 +5,6 @@ import ( "log" "time" - "github.com/krisukox/google-flights-api/api" "golang.org/x/text/currency" ) @@ -13,7 +12,7 @@ func main() { date, _ := time.Parse("2006-01-02", "2023-07-04") returnDate, _ := time.Parse("2006-01-02", "2023-07-08") - offer, err := api.GetOffers(date, returnDate, "Wrocław", "Rzym", currency.PLN) + offer, err := GetOffers(date, returnDate, "Wrocław", "Rzym", currency.PLN) if err != nil { log.Fatalf(err.Error()) } diff --git a/api/serialize_city.go b/serialize_city.go similarity index 93% rename from api/serialize_city.go rename to serialize_city.go index 9661a7d..ac00a0f 100644 --- a/api/serialize_city.go +++ b/serialize_city.go @@ -1,4 +1,4 @@ -package api +package main import ( "bufio" @@ -88,7 +88,7 @@ func decodeInnerObject(body *bufio.Reader) ([][][][]interface{}, error) { return innerObject, nil } -func GetSerializedCityName(city string) (string, error) { +func AbbrCity(city string) (string, error) { resp, err := sendRequest(city) if err != nil { return "", err @@ -117,3 +117,15 @@ func GetSerializedCityName(city string) (string, error) { return serializedCity, nil } + +func AbbrCities(cities []string) ([]string, error) { + abbrCities := []string{} + for _, c := range cities { + sc, err := AbbrCity(c) + if err != nil { + return nil, err + } + abbrCities = append(abbrCities, sc) + } + return abbrCities, nil +} diff --git a/url.go b/url.go new file mode 100644 index 0000000..0ca164d --- /dev/null +++ b/url.go @@ -0,0 +1,201 @@ +package main + +import ( + "encoding/base64" + "fmt" + "time" + + "golang.org/x/text/currency" +) + +const ( + airportConst byte = 1 + cityConst byte = 3 + dstConst byte = 114 + srcConst byte = 106 +) + +type Stops int64 + +const ( + AnyStops Stops = iota + Nonstop + Stop1 + Stop2 +) + +type Class int64 + +const ( + Economy Class = iota + PremiumEconomy + Buisness + First +) + +type TripType int64 + +const ( + RoundTrip TripType = iota + OneWay +) + +func checkMaxLocations(cities, airports []string) (bool, error) { + length := len(cities) + len(airports) + if length <= 7 { + return true, nil + } + return false, fmt.Errorf("specified number of locations (%d) should not exceed 7", length) +} + +func serializeLocation(city string, locationNo byte) []byte { + cityBytes := []byte(city) + bytes := append([]byte{8, locationNo, 18}, append([]byte{byte(len(cityBytes))}, cityBytes...)...) + return append([]byte{byte(len(bytes))}, bytes...) +} + +func serializeSrcCity(city string) []byte { + return append([]byte{srcConst}, serializeLocation(city, cityConst)...) +} + +func serializeDstCity(city string) []byte { + return append([]byte{dstConst}, serializeLocation(city, cityConst)...) +} + +func serializeSrcAirport(airport string) []byte { + return append([]byte{srcConst}, serializeLocation(airport, airportConst)...) +} + +func serializeDstAirport(airport string) []byte { + return append([]byte{dstConst}, serializeLocation(airport, airportConst)...) +} + +func serializeLocations(locations []string, f func(string) []byte) []byte { + ret := []byte{} + for _, l := range locations { + ret = append(ret, f(l)...) + } + return ret +} + +func serializeDate(date time.Time) []byte { + return append([]byte{18, 10}, []byte(date.Format("2006-01-02"))...) +} + +func serializeStops(stops Stops) []byte { + switch stops { + case Nonstop: + return []byte{40, 0} + case Stop1: + return []byte{40, 1} + case Stop2: + return []byte{40, 1} + } + return []byte{} +} + +func serializeClass(class Class) []byte { + switch class { + case Economy: + return []byte{72, 1} + case PremiumEconomy: + return []byte{72, 2} + case Buisness: + return []byte{72, 3} + } + return []byte{72, 4} +} + +func serializeFlight( + date time.Time, + srcCities []string, + srcAirports []string, + dstCities []string, + dstAirports []string, + stops Stops, +) []byte { + bytes := serializeDate(date) + bytes = append(bytes, serializeStops(stops)...) + bytes = append(bytes, serializeLocations(srcCities, serializeSrcCity)...) + bytes = append(bytes, serializeLocations(srcAirports, serializeSrcAirport)...) + bytes = append(bytes, serializeLocations(dstCities, serializeDstCity)...) + bytes = append(bytes, serializeLocations(dstAirports, serializeDstAirport)...) + return append([]byte{26, byte(len(bytes))}, bytes...) +} + +func serializeAdults(adults int) []byte { + bytes := []byte{} + for i := 0; i < adults; i++ { + bytes = append(bytes, 64, 1) + } + return bytes +} + +func serializeTripType(tripType TripType) byte { + if tripType == RoundTrip { + return 1 + } + return 2 +} + +func SerializeUrl( + date time.Time, + returnDate time.Time, + srcCities []string, + srcAirports []string, + dstCities []string, + dstAirports []string, + adults int, + curr currency.Unit, + stops Stops, + class Class, + tripType TripType, +) (string, error) { + if ok, err := checkMaxLocations(srcCities, srcAirports); !ok { + return "", fmt.Errorf("sources: %s", err.Error()) + } + + if ok, err := checkMaxLocations(dstCities, dstAirports); !ok { + return "", fmt.Errorf("destinations: %s", err.Error()) + } + + srcCities, err := AbbrCities(srcCities) + if err != nil { + return "", err + } + + dstCities, err = AbbrCities(dstCities) + if err != nil { + return "", err + } + + bytes := []byte{8, 28, 16, 2} + + additionalBytes := []byte{ + 112, 1, + 130, 1, + 11, + 8, 255, 255, 255, 255, 255, 255, 255, 255, 255, 1, + 152, 1, + } + + bytes = append(bytes, serializeFlight(date, srcCities, srcAirports, dstCities, dstAirports, stops)...) + + if tripType == RoundTrip { + bytes = append(bytes, serializeFlight(returnDate, dstCities, dstAirports, srcCities, srcAirports, stops)...) + } + + bytes = append(bytes, serializeAdults(adults)...) + + bytes = append(bytes, serializeClass(class)...) + + bytes = append(bytes, additionalBytes...) + + bytes = append(bytes, serializeTripType(tripType)) + + RawURLEncoding := base64.URLEncoding.WithPadding(base64.NoPadding) + + url := "https://www.google.com/travel/flights/search?tfs=" + RawURLEncoding.EncodeToString(bytes) + "&curr=" + curr.String() + + return url, nil +} diff --git a/url_test.go b/url_test.go new file mode 100644 index 0000000..23fae96 --- /dev/null +++ b/url_test.go @@ -0,0 +1,112 @@ +package main + +import ( + "bytes" + "testing" + "time" + + "golang.org/x/text/currency" +) + +func TestSerializeSrcCity(t *testing.T) { + expectedSerializedCity1 := []byte{ + 106, 12, + 8, 3, + 18, 8, + 47, 109, 47, 48, 56, 52, 53, 98} + serializedCity1 := serializeSrcCity("/m/0845b") + + if !bytes.Equal(expectedSerializedCity1, serializedCity1) { + t.Fatalf("wrong serialized source city, expected: %v serialized: %v", expectedSerializedCity1, serializedCity1) + } + + expectedSerializedCity2 := []byte{ + 106, 11, + 8, 3, + 18, 7, + 47, 109, 47, 48, 110, 50, 122} + serializedCity2 := serializeSrcCity("/m/0n2z") + + if !bytes.Equal(expectedSerializedCity2, serializedCity2) { + t.Fatalf("wrong serialized source city, expected: %v serialized: %v", expectedSerializedCity2, serializedCity2) + } +} + +func TestSerializeDstCity(t *testing.T) { + expectedSerializedCity1 := []byte{ + 114, 12, + 8, 3, + 18, 8, + 47, 109, 47, 48, 56, 52, 53, 98} + serializedCity1 := serializeDstCity("/m/0845b") + + if !bytes.Equal(expectedSerializedCity1, serializedCity1) { + t.Fatalf("wrong serialized city, expected: %v serialized: %v", expectedSerializedCity1, serializedCity1) + } + + expectedSerializedSrcCity2 := []byte{ + 114, 11, + 8, 3, + 18, 7, + 47, 109, 47, 48, 110, 50, 122} + serializedCity2 := serializeDstCity("/m/0n2z") + + if !bytes.Equal(expectedSerializedSrcCity2, serializedCity2) { + t.Fatalf("wrong serialized city, expected: %v serialized: %v", expectedSerializedSrcCity2, serializedCity2) + } +} + +func TestSerializeSrcAirport(t *testing.T) { + expectedSerializedCity := []byte{ + 106, 7, + 8, 1, + 18, 3, + 76, 67, 89} + serializedCity := serializeSrcAirport("LCY") + + if !bytes.Equal(expectedSerializedCity, serializedCity) { + t.Fatalf("wrong serialized city, expected: %v serialized: %v", serializedCity, serializedCity) + } +} + +func TestSerializeDstAirport(t *testing.T) { + expectedSerializedCity := []byte{ + 114, 7, + 8, 1, + 18, 3, + 76, 67, 89} + serializedCity := serializeDstAirport("LCY") + + if !bytes.Equal(expectedSerializedCity, serializedCity) { + t.Fatalf("wrong serialized city, expected: %v serialized: %v", serializedCity, serializedCity) + } +} + +func TestSerializeUrl(t *testing.T) { + expectedUrl := "https://www.google.com/travel/flights/search?tfs=CBwQAho-EgoyMDIzLTExLTA2KAFqDggDEgovbS8wMzBxYjN0agcIARIDU0ZPcgwIAxIIL20vMDRqcGxyBwgBEgNDREcaPhIKMjAyMy0xMS0xMygBagwIAxIIL20vMDRqcGxqBwgBEgNDREdyDggDEgovbS8wMzBxYjN0cgcIARIDU0ZPQAFAAUgBcAGCAQsI____________AZgBAQ&curr=USD" + + date, _ := time.Parse("2006-01-02", "2023-11-06") + returnDate, _ := time.Parse("2006-01-02", "2023-11-13") + + url, err := SerializeUrl( + date, + returnDate, + []string{"Los Angeles"}, + []string{"SFO"}, + []string{"Londyn"}, + []string{"CDG"}, + 2, + currency.USD, + Stop1, + Economy, + RoundTrip, + ) + + if err != nil { + t.Fatalf("error during serialization: %s", err.Error()) + } + + if expectedUrl != url { + t.Fatalf("wrong serialized url, expected: %v serialized: %v", expectedUrl, url) + } +}