Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Made OSM header accessible #24

Merged
merged 6 commits into from
Jun 22, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 78 additions & 12 deletions decode.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"errors"
"fmt"
"io"
"sync"
"time"

"github.com/golang/protobuf/proto"
Expand All @@ -33,6 +34,24 @@ var (
}
)

type BoundingBox struct {
Left float64
Right float64
Top float64
Bottom float64
}

type Header struct {
BoundingBox *BoundingBox
RequiredFeatures []string
OptionalFeatures []string
WritingProgram string
Source string
OsmosisReplicationTimestamp time.Time
OsmosisReplicationSequenceNumber int64
OsmosisReplicationBaseUrl string
}

type Info struct {
Version int32
Timestamp time.Time
Expand Down Expand Up @@ -90,6 +109,11 @@ type Decoder struct {

buf *bytes.Buffer

// store header block
header *Header
// synchronize header deserialization
headerOnce sync.Once

// for data decoders
inputs []chan<- pair
outputs []<-chan pair
Expand All @@ -112,22 +136,19 @@ func (dec *Decoder) SetBufferSize(n int) {
dec.buf = bytes.NewBuffer(make([]byte, 0, n))
}

// Get the file header
func (dec *Decoder) Header() (*Header, error) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe this may be simplified to the following without any worries about race conditions:

func (dec *Decoder) Header() (*Header, error) {
    return dec.header, dec.readOSMHeader()
}

The dec.header pointer should always be passed to the calling function after dec.readOSMHeader() has been executed so dec.header will be set unless dec.readOSMHeader() produces an error. The down side of my suggestion is that the race condition, or lack thereof, is less explicit.

// deserialize the file header
return dec.header, dec.readOSMHeader()
}

// Start decoding process using n goroutines.
func (dec *Decoder) Start(n int) error {
if n < 1 {
n = 1
}

// read OSMHeader
blobHeader, blob, err := dec.readFileBlock()
if err == nil {
if blobHeader.GetType() == "OSMHeader" {
err = decodeOSMHeader(blob)
} else {
err = fmt.Errorf("unexpected first fileblock of type %s", blobHeader.GetType())
}
}
if err != nil {
if err := dec.readOSMHeader(); err != nil {
return err
}

Expand Down Expand Up @@ -161,7 +182,7 @@ func (dec *Decoder) Start(n int) error {
input := dec.inputs[inputIndex]
inputIndex = (inputIndex + 1) % n

blobHeader, blob, err = dec.readFileBlock()
blobHeader, blob, err := dec.readFileBlock()
if err == nil && blobHeader.GetType() != "OSMData" {
err = fmt.Errorf("unexpected fileblock of type %s", blobHeader.GetType())
}
Expand Down Expand Up @@ -307,7 +328,25 @@ func getData(blob *OSMPBF.Blob) ([]byte, error) {
}
}

func decodeOSMHeader(blob *OSMPBF.Blob) error {
func (dec *Decoder) readOSMHeader() error {
var err error
dec.headerOnce.Do(func() {
var blobHeader *OSMPBF.BlobHeader
var blob *OSMPBF.Blob
blobHeader, blob, err = dec.readFileBlock()
if err == nil {
if blobHeader.GetType() == "OSMHeader" {
err = dec.decodeOSMHeader(blob)
} else {
err = fmt.Errorf("unexpected first fileblock of type %s", blobHeader.GetType())
}
}
})

return err
}

func (dec *Decoder) decodeOSMHeader(blob *OSMPBF.Blob) error {
data, err := getData(blob)
if err != nil {
return err
Expand All @@ -326,5 +365,32 @@ func decodeOSMHeader(blob *OSMPBF.Blob) error {
}
}

// Read properties to header struct
header := &Header{
RequiredFeatures: headerBlock.GetRequiredFeatures(),
OptionalFeatures: headerBlock.GetOptionalFeatures(),
WritingProgram: headerBlock.GetWritingprogram(),
Source: headerBlock.GetSource(),
OsmosisReplicationBaseUrl: headerBlock.GetOsmosisReplicationBaseUrl(),
OsmosisReplicationSequenceNumber: headerBlock.GetOsmosisReplicationSequenceNumber(),
}

// convert timestamp epoch seconds to golang time structure if it exists
if headerBlock.OsmosisReplicationTimestamp != nil {
header.OsmosisReplicationTimestamp = time.Unix(*headerBlock.OsmosisReplicationTimestamp, 0)
}
// read bounding box if it exists
if headerBlock.Bbox != nil {
// Units are always in nanodegree and do not obey granularity rules. See osmformat.proto
header.BoundingBox = &BoundingBox{
Left: 1e-9 * float64(*headerBlock.Bbox.Left),
Right: 1e-9 * float64(*headerBlock.Bbox.Right),
Bottom: 1e-9 * float64(*headerBlock.Bbox.Bottom),
Top: 1e-9 * float64(*headerBlock.Bbox.Top),
}
}

dec.header = header

return nil
}
60 changes: 60 additions & 0 deletions decode_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,21 @@ var (
ewc uint64 = 459055
erc uint64 = 12833

eh = &Header{
BoundingBox: &BoundingBox{
Right: 0.335437,
Left: -0.511482,
Bottom: 51.28554,
Top: 51.69344,
},
OsmosisReplicationTimestamp: time.Date(2014, 3, 24, 22, 55, 2, 0, time.FixedZone("test", 3600)),
RequiredFeatures: []string{
"OsmSchema-V0.6",
"DenseNodes",
},
WritingProgram: `Osmium (http:\/\/wiki.openstreetmap.org\/wiki\/Osmium)`,
}

en = &Node{
ID: 18088578,
Lat: 51.5442632,
Expand Down Expand Up @@ -152,6 +167,34 @@ func downloadTestOSMFile(t *testing.T) {
}
}

func checkHeader(a *Header) bool {
if a == nil || a.BoundingBox == nil || a.RequiredFeatures == nil {
return false
}

// check bbox
if a.BoundingBox.Right != eh.BoundingBox.Right || a.BoundingBox.Left != eh.BoundingBox.Left || a.BoundingBox.Top != eh.BoundingBox.Top || a.BoundingBox.Bottom != eh.BoundingBox.Bottom {
return false
}

// check timestamp
if !a.OsmosisReplicationTimestamp.Equal(eh.OsmosisReplicationTimestamp) {
return false
}

// check writing program
if a.WritingProgram != eh.WritingProgram {
return false
}

// check features
if len(a.RequiredFeatures) != len(eh.RequiredFeatures) || a.RequiredFeatures[0] != eh.RequiredFeatures[0] || a.RequiredFeatures[1] != eh.RequiredFeatures[1] {
return false
}

return true
}

func TestDecode(t *testing.T) {
downloadTestOSMFile(t)

Expand All @@ -163,6 +206,15 @@ func TestDecode(t *testing.T) {

d := NewDecoder(f)
d.SetBufferSize(1)

header, err := d.Header()
if err != nil {
t.Fatal(err)
}
if checkHeader(header) {
t.Errorf("\nExpected: %#v\nActual: %#v", eh, header)
}

err = d.Start(runtime.GOMAXPROCS(-1))
if err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -248,6 +300,14 @@ func TestDecodeConcurrent(t *testing.T) {
t.Fatal(err)
}

header, err := d.Header()
if err != nil {
t.Fatal(err)
}
if checkHeader(header) {
t.Errorf("\nExpected: %#v\nActual: %#v", eh, header)
}

var n *Node
var w *Way
var r *Relation
Expand Down