Skip to content

Commit

Permalink
BurstsFromAnnotation => interface folder
Browse files Browse the repository at this point in the history
  • Loading branch information
vincentvaroquauxads committed Jan 22, 2024
1 parent d77afbb commit b72d2c8
Show file tree
Hide file tree
Showing 5 changed files with 106 additions and 87 deletions.
98 changes: 16 additions & 82 deletions catalog/bursts.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,15 @@ package catalog

import (
"context"
"encoding/xml"
"fmt"
"math"
"runtime"
"sort"
"time"

"github.com/airbusgeo/geocube-ingester/catalog/entities"
"github.com/airbusgeo/geocube-ingester/common"
"github.com/airbusgeo/geocube-ingester/interface/catalog"
"github.com/airbusgeo/geocube-ingester/interface/catalog/annotations"
"github.com/airbusgeo/geocube-ingester/interface/catalog/url"
"github.com/airbusgeo/geocube-ingester/service"
"github.com/airbusgeo/geocube-ingester/service/log"
Expand Down Expand Up @@ -172,7 +171,7 @@ func burstsFromAnnotations(ctx context.Context, scene *entities.Scene, annotatio
var err, e error
var annotationsFiles map[string][]byte
for _, annotationsProvider := range annotationsProviders {
annotationsFiles, e = annotationsProvider.AnnotationsFiles(ctx, scene)
annotationsFiles, e = annotationsProvider.AnnotationsFiles(ctx, &scene.Scene)
if err = service.MergeErrors(false, err, e); err == nil {
break
}
Expand All @@ -184,7 +183,7 @@ func burstsFromAnnotations(ctx context.Context, scene *entities.Scene, annotatio
var burstsInventory []*entities.Tile
anxTimes := map[int]struct{}{}
for url, file := range annotationsFiles {
bursts, err := burstsFromAnnotation(file, url)
bursts, err := annotations.BurstsFromAnnotation(file, url)
if err != nil {
return nil, fmt.Errorf("burstsFromAnnotations.%w", err)
}
Expand All @@ -193,87 +192,22 @@ func burstsFromAnnotations(ctx context.Context, scene *entities.Scene, annotatio
anxTimes[anxTime] = struct{}{}

// Add info from scene
burst.Date = scene.Scene.Data.Date
burst.SceneID = scene.SourceID
burst.SourceID = fmt.Sprintf("%s%s_%s_%d", scene.Tags[common.TagOrbitDirection][0:1], scene.Tags[common.TagRelativeOrbit], burst.Data.SwathID, burst.AnxTime)
burstsInventory = append(burstsInventory, burst)
burstsInventory = append(burstsInventory, &entities.Tile{
TileLite: entities.TileLite{
Date: scene.Data.Date,
SceneID: scene.SourceID,
SourceID: fmt.Sprintf("%s%s_%s_%d", scene.Tags[common.TagOrbitDirection][0:1], scene.Tags[common.TagRelativeOrbit], burst.SwathID, burst.AnxTime),
},
Data: common.TileAttrs{
SwathID: burst.SwathID,
TileNr: burst.TileNr,
},
AnxTime: burst.AnxTime,
GeometryWKT: burst.GeometryWKT,
})
}
}
}

return burstsInventory, nil
}

func burstsFromAnnotation(annotationFile []byte, annotationURL string) (map[int]*entities.Tile, error) {
// XML GridPoint structure
type GridPoint struct {
Pixel int `xml:"pixel"`
Line int `xml:"line"`
Latitude float64 `xml:"latitude"`
Longitude float64 `xml:"longitude"`
}

// Read annotations file
annotation := struct {
XMLName xml.Name `xml:"product"`
Swath string `xml:"adsHeader>swath"`
LinesPerBurst int `xml:"swathTiming>linesPerBurst"`
SamplesPerBurst int `xml:"swathTiming>samplesPerBurst"`
AzimuthAnxTime []float64 `xml:"swathTiming>burstList>burst>azimuthAnxTime"`

GridPoint []GridPoint `xml:"geolocationGrid>geolocationGridPointList>geolocationGridPoint"`
}{}
if err := xml.Unmarshal(annotationFile, &annotation); err != nil {
return nil, fmt.Errorf("readAnnotation.Unmarshal[%s] : %w", annotationFile, err)
}

// Position of firsts and last points
first := map[int]GridPoint{}
last := map[int]GridPoint{}
for _, point := range annotation.GridPoint {
if point.Pixel == 0 {
first[point.Line] = point
} else if point.Pixel == annotation.SamplesPerBurst-1 {
last[point.Line] = point
}
}

// Burst
bursts := map[int]*entities.Tile{}
for i, anxTime := range annotation.AzimuthAnxTime {
// First/Last lines of the burst
firstline := i * annotation.LinesPerBurst
lastline := (i + 1) * annotation.LinesPerBurst
if _, ok := first[firstline]; !ok {
firstline-- // -1 because first and lastline sometimes shifts by 1 for some reason?
if _, ok := first[firstline]; !ok {
return nil, fmt.Errorf("readAnnotation: First line not found in annotation file %s", annotationURL)
}
}
if _, ok := last[lastline]; !ok {
lastline-- // -1 because first and lastline sometimes shifts by 1 for some reason?
if _, ok := last[lastline]; !ok {
return nil, fmt.Errorf("readAnnotation: Last line not found in annotation file %s", annotationURL)
}
}

// Set bursts
intAnxTime := int(math.Round(math.Mod(anxTime, float64(12*24*60*60/175)) * 10))
bursts[intAnxTime] = &entities.Tile{
Data: common.TileAttrs{
SwathID: annotation.Swath,
TileNr: i + 1,
},
AnxTime: intAnxTime,
GeometryWKT: fmt.Sprintf("POLYGON((%f %f, %f %f, %f %f, %f %f, %f %f))",
first[firstline].Longitude, first[firstline].Latitude,
first[lastline].Longitude, first[lastline].Latitude,
last[lastline].Longitude, last[lastline].Latitude,
last[firstline].Longitude, last[firstline].Latitude,
first[firstline].Longitude, first[firstline].Latitude,
),
}
}

return bursts, nil
}
86 changes: 86 additions & 0 deletions interface/catalog/annotations/annotationFile.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
package annotations

import (
"encoding/xml"
"fmt"
"math"
)

type Burst struct {
SwathID string
TileNr int
AnxTime int
GeometryWKT string
}

func BurstsFromAnnotation(annotationFile []byte, annotationURL string) (map[int]*Burst, error) {
// XML GridPoint structure
type GridPoint struct {
Pixel int `xml:"pixel"`
Line int `xml:"line"`
Latitude float64 `xml:"latitude"`
Longitude float64 `xml:"longitude"`
}

// Read annotations file
annotation := struct {
XMLName xml.Name `xml:"product"`
Swath string `xml:"adsHeader>swath"`
LinesPerBurst int `xml:"swathTiming>linesPerBurst"`
SamplesPerBurst int `xml:"swathTiming>samplesPerBurst"`
AzimuthAnxTime []float64 `xml:"swathTiming>burstList>burst>azimuthAnxTime"`

GridPoint []GridPoint `xml:"geolocationGrid>geolocationGridPointList>geolocationGridPoint"`
}{}
if err := xml.Unmarshal(annotationFile, &annotation); err != nil {
return nil, fmt.Errorf("readAnnotation.Unmarshal[%s] : %w", annotationFile, err)
}

// Position of firsts and last points
first := map[int]GridPoint{}
last := map[int]GridPoint{}
for _, point := range annotation.GridPoint {
if point.Pixel == 0 {
first[point.Line] = point
} else if point.Pixel == annotation.SamplesPerBurst-1 {
last[point.Line] = point
}
}

// Burst
bursts := map[int]*Burst{}
for i, anxTime := range annotation.AzimuthAnxTime {
// First/Last lines of the burst
firstline := i * annotation.LinesPerBurst
lastline := (i + 1) * annotation.LinesPerBurst
if _, ok := first[firstline]; !ok {
firstline-- // -1 because first and lastline sometimes shifts by 1 for some reason?
if _, ok := first[firstline]; !ok {
return nil, fmt.Errorf("readAnnotation: First line not found in annotation file %s", annotationURL)
}
}
if _, ok := last[lastline]; !ok {
lastline-- // -1 because first and lastline sometimes shifts by 1 for some reason?
if _, ok := last[lastline]; !ok {
return nil, fmt.Errorf("readAnnotation: Last line not found in annotation file %s", annotationURL)
}
}

// Set bursts
intAnxTime := int(math.Round(math.Mod(anxTime, float64(12*24*60*60/175)) * 10))
bursts[intAnxTime] = &Burst{
SwathID: annotation.Swath,
TileNr: i + 1,
AnxTime: intAnxTime,
GeometryWKT: fmt.Sprintf("POLYGON((%f %f, %f %f, %f %f, %f %f, %f %f))",
first[firstline].Longitude, first[firstline].Latitude,
first[lastline].Longitude, first[lastline].Latitude,
last[lastline].Longitude, last[lastline].Latitude,
last[firstline].Longitude, last[firstline].Latitude,
first[firstline].Longitude, first[firstline].Latitude,
),
}
}

return bursts, nil
}
3 changes: 2 additions & 1 deletion interface/catalog/catalog.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@ import (
"context"

"github.com/airbusgeo/geocube-ingester/catalog/entities"
"github.com/airbusgeo/geocube-ingester/common"
"github.com/paulsmith/gogeos/geos"
)

type AnnotationsProvider interface {
AnnotationsFiles(ctx context.Context, scene *entities.Scene) (map[string][]byte, error)
AnnotationsFiles(ctx context.Context, scene *common.Scene) (map[string][]byte, error)
}

type ScenesProvider interface {
Expand Down
3 changes: 1 addition & 2 deletions interface/catalog/gcs/bursts.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"path/filepath"
"regexp"

"github.com/airbusgeo/geocube-ingester/catalog/entities"
"github.com/airbusgeo/geocube-ingester/common"
"github.com/airbusgeo/osio"
osioGcs "github.com/airbusgeo/osio/gcs"
Expand All @@ -19,7 +18,7 @@ type AnnotationsProvider struct {
}

// AnnotationFiles retrieves them from an archive in GCS
func (ap AnnotationsProvider) AnnotationsFiles(ctx context.Context, scene *entities.Scene) (map[string][]byte, error) {
func (ap AnnotationsProvider) AnnotationsFiles(ctx context.Context, scene *common.Scene) (map[string][]byte, error) {
reg, err := regexp.Compile(scene.SourceID + ".SAFE/annotation/s1[^/]*xml")
if err != nil {
return nil, fmt.Errorf("annotationFiles.Compile[%s]: %w", scene.SourceID+".SAFE/annotation/*xml", err)
Expand Down
3 changes: 1 addition & 2 deletions interface/catalog/url/annotations.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
osioGcs "github.com/airbusgeo/osio/gcs"
osioS3 "github.com/airbusgeo/osio/s3"

"github.com/airbusgeo/geocube-ingester/catalog/entities"
"github.com/airbusgeo/geocube-ingester/common"
"github.com/airbusgeo/geocube/interface/storage/uri"
"github.com/airbusgeo/osio"
Expand All @@ -25,7 +24,7 @@ type AnnotationsProvider struct {
}

// AnnotationFiles retrieves them from an archive in GCS
func (ap AnnotationsProvider) AnnotationsFiles(ctx context.Context, scene *entities.Scene) (map[string][]byte, error) {
func (ap AnnotationsProvider) AnnotationsFiles(ctx context.Context, scene *common.Scene) (map[string][]byte, error) {
reg, err := regexp.Compile(scene.SourceID + ".SAFE/annotation/s1[^/]*xml")
if err != nil {
return nil, fmt.Errorf("annotationFiles.Compile[%s]: %w", scene.SourceID+".SAFE/annotation/*xml", err)
Expand Down

0 comments on commit b72d2c8

Please sign in to comment.