diff --git a/server/datacatalog/citygml.go b/server/datacatalog/citygml.go index b3904737f..2435d8968 100644 --- a/server/datacatalog/citygml.go +++ b/server/datacatalog/citygml.go @@ -2,10 +2,7 @@ package datacatalog import ( "context" - "encoding/csv" "fmt" - "io" - "net/http" "net/url" "path" "slices" @@ -16,7 +13,6 @@ import ( "github.com/eukarya-inc/reearth-plateauview/server/plateaucms" "github.com/reearth/reearthx/util" "github.com/samber/lo" - "github.com/spkg/bom" ) type CityGMLFilesResponse struct { @@ -73,8 +69,13 @@ func fetchCityGMLFiles(ctx context.Context, r plateauapi.Repo, id string) (*City return nil, nil } - maxlodURL := admin["maxlod"].(string) - if maxlodURL == "" { + maxlodURLs, ok := admin["maxlod"].([]string) + if !ok { + return nil, nil + } + + citygmlURLs, ok := admin["citygmlUrl"].([]string) + if !ok { return nil, nil } @@ -106,12 +107,12 @@ func fetchCityGMLFiles(ctx context.Context, r plateauapi.Repo, id string) (*City gurls = gmlURLs(asset.File.Paths(), assetBase) } - data, err := fetchCSV(ctx, maxlodURL) + data, err := fetchCSVs(ctx, maxlodURLs, citygmlURLs) if err != nil { return nil, err } - files := csvToCityGMLFilesResponse(data, citygml.URL, gurls) + files := csvToCityGMLFilesResponse(data, gurls) return &CityGMLFilesResponse{ CityCode: string(citygml.CityCode), CityName: city.Name, @@ -123,59 +124,28 @@ func fetchCityGMLFiles(ctx context.Context, r plateauapi.Repo, id string) (*City }, nil } -func fetchCSV(ctx context.Context, url string) (records [][]string, _ error) { - res, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - resp, err := http.DefaultClient.Do(res) - if err != nil { - return nil, fmt.Errorf("failed to request: %w", err) - } - - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to request: %w", err) - } - - c := csv.NewReader(bom.NewReader(resp.Body)) - for { - record, err := c.Read() - if err == io.EOF { - break - } - if err != nil { - return nil, fmt.Errorf("failed to read csv: %w", err) - } - records = append(records, record) - } - - return -} - -func csvToCityGMLFilesResponse(data [][]string, base string, gmlURLs []*url.URL) CityGMLFiles { +func csvToCityGMLFilesResponse(data [][]string, gmlURLs []*url.URL) CityGMLFiles { res := make(CityGMLFiles) for _, record := range data { - if len(record) < 3 || record[0] == "" { + if len(record) < 3 || record[0] == "" || record[1] == "" { continue } - if !isNumeric(rune(record[0][0])) { - // it's a header - continue + if !isNumeric(rune(record[1][0])) { + continue // skip header } - // code,type,maxLod(,path) - meshCode := record[0] - featureType := record[1] - maxlod, _ := strconv.Atoi(record[2]) + // base,code,type,maxLod(,path) + base := record[0] + meshCode := record[1] + featureType := record[2] + maxlod, _ := strconv.Atoi(record[3]) citygmlURL := "" + gmlPath := record[4] - if len(record) > 3 && gmlURLs == nil { - citygmlURL = citygmlItemURLFrom(base, record[3], featureType) + if len(record) > 4 && gmlURLs == nil { + citygmlURL = citygmlItemURLFrom(base, gmlPath, featureType) } else { // compat for datacatalogv2 prefix := fmt.Sprintf("%s_%s_", meshCode, featureType) diff --git a/server/datacatalog/citygml_csv.go b/server/datacatalog/citygml_csv.go new file mode 100644 index 000000000..e027631fa --- /dev/null +++ b/server/datacatalog/citygml_csv.go @@ -0,0 +1,79 @@ +package datacatalog + +import ( + "context" + "encoding/csv" + "fmt" + "io" + "net/http" + + "github.com/reearth/reearthx/rerror" + "github.com/spkg/bom" + "golang.org/x/sync/errgroup" +) + +func fetchCSVs(ctx context.Context, urls, citygmlBaseURLs []string) (records [][]string, _ error) { + if len(urls) != len(citygmlBaseURLs) { + return nil, fmt.Errorf("length of urls and citygmlBaseURLs must be the same") + } + + errg := errgroup.Group{} + errg.SetLimit(10) + + for i, url := range urls { + url := url + base := citygmlBaseURLs[i] + errg.Go(func() error { + data, err := fetchCSV(ctx, url, base) + if err != nil { + return fmt.Errorf("failed to fetch %s: %w", url, err) + } + + records = append(records, data...) + return nil + }) + } + + if err := errg.Wait(); err != nil { + return nil, err + } + + return records, nil +} + +func fetchCSV(ctx context.Context, url, prefix string) (records [][]string, _ error) { + res, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + resp, err := http.DefaultClient.Do(res) + if err != nil { + return nil, fmt.Errorf("failed to request: %w", err) + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusNotFound { + return nil, rerror.ErrNotFound + } + return nil, fmt.Errorf("failed to request: %w", err) + } + + c := csv.NewReader(bom.NewReader(resp.Body)) + for { + record, err := c.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("failed to read csv: %w", err) + } + + record = append([]string{prefix}, record...) + records = append(records, record) + } + + return +} diff --git a/server/datacatalog/datacatalogv2/datacatalogv2adapter/cache_test.go b/server/datacatalog/datacatalogv2/datacatalogv2adapter/cache_test.go index 988be17c0..dedcd0ecc 100644 --- a/server/datacatalog/datacatalogv2/datacatalogv2adapter/cache_test.go +++ b/server/datacatalog/datacatalogv2/datacatalogv2adapter/cache_test.go @@ -133,7 +133,8 @@ func TestNewCache(t *testing.T) { URL: "https://example.com/13101_tokyo23ku_2022_citygml_op_2.zip", FeatureTypes: []string{"bldg", "dem"}, Admin: map[string]any{ - "maxlod": "maxlod2", + "citygmlUrl": []string{"https://example.com/13101_tokyo23ku_2022_citygml_op_2.zip"}, + "maxlod": []string{"maxlod2"}, "citygmlAssetId": "assetid2", }, }, diff --git a/server/datacatalog/datacatalogv2/datacatalogv2adapter/type.go b/server/datacatalog/datacatalogv2/datacatalogv2adapter/type.go index 27f9164c3..21f70c15c 100644 --- a/server/datacatalog/datacatalogv2/datacatalogv2adapter/type.go +++ b/server/datacatalog/datacatalogv2/datacatalogv2adapter/type.go @@ -517,7 +517,8 @@ func citygmlFrom(d datacatalogv2.DataCatalogItem, i *fetcherPlateauItem2) *plate URL: i.CityGMLURL, FeatureTypes: i.FeatureTypes, Admin: map[string]any{ - "maxlod": i.MaxLODURL, + "maxlod": []string{i.MaxLODURL}, + "citygmlUrl": []string{i.CityGMLURL}, "citygmlAssetId": i.CityGMLAssetID, }, } diff --git a/server/datacatalog/datacatalogv3/cms_model.go b/server/datacatalog/datacatalogv3/cms_model.go index d3d545037..891d8c3b1 100644 --- a/server/datacatalog/datacatalogv3/cms_model.go +++ b/server/datacatalog/datacatalogv3/cms_model.go @@ -123,6 +123,9 @@ func (i *CityItem) SDKStage() stage { if i.SDKPublic { return stageGA } + if i.PlateauStage("") == stageBeta { + return stageBeta + } return stageAlpha } @@ -151,12 +154,7 @@ type PlateauFeatureItem struct { Dic string `json:"dic,omitempty" cms:"dic,textarea"` MaxLOD string `json:"maxlod,omitempty" cms:"maxlod,-"` // metadata - Status *cms.Tag `json:"status,omitempty" cms:"status,select,metadata"` - Sample bool `json:"sample,omitempty" cms:"sample,bool,metadata"` -} - -func (c PlateauFeatureItem) IsPublicForAdmin() bool { - return c.Status != nil && c.Status.Name == string(ManagementStatusReady) + Sample bool `json:"sample,omitempty" cms:"sample,bool,metadata"` } func (c PlateauFeatureItem) ReadDic() (d Dic, _ error) { @@ -432,11 +430,10 @@ func geospatialjpURL(cityCode string, cityName string, year int) string { } type GeospatialjpDataItem struct { - ID string `json:"id,omitempty" cms:"id"` - City string `json:"city,omitempty" cms:"city,reference"` - CityGML string `json:"citygml,omitempty" cms:"citygml,asset"` - MaxLOD string `json:"maxlod,omitempty" cms:"maxlod,asset"` - MaxLODContent [][]string `json:"maxlod_content,omitempty" cms:"-"` + ID string `json:"id,omitempty" cms:"id"` + City string `json:"city,omitempty" cms:"city,reference"` + CityGML string `json:"citygml,omitempty" cms:"citygml,asset"` + MaxLOD string `json:"maxlod,omitempty" cms:"maxlod,asset"` } func GeospatialjpDataItemFrom(item *cms.Item) *GeospatialjpDataItem { diff --git a/server/datacatalog/datacatalogv3/conv_citygml.go b/server/datacatalog/datacatalogv3/conv_citygml.go index 037f79a98..36d34b8cc 100644 --- a/server/datacatalog/datacatalogv3/conv_citygml.go +++ b/server/datacatalog/datacatalogv3/conv_citygml.go @@ -5,45 +5,76 @@ import ( ) func toCityGMLs(all *AllData, regYear int) (map[plateauapi.ID]*plateauapi.CityGMLDataset, []string) { - cities := all.City - data := all.GeospatialjpDataItems cmsurl := all.CMSInfo.CMSURL + res := map[plateauapi.ID]*plateauapi.CityGMLDataset{} + resCity := map[string]*plateauapi.CityGMLDataset{} + dataMap := make(map[string]*GeospatialjpDataItem) + cityMap := make(map[string]*CityItem) - dataMap := make(map[string]*plateauapi.CityGMLDataset) - - for _, d := range data { + for _, d := range all.GeospatialjpDataItems { if d.CityGML == "" || d.MaxLOD == "" { continue } - - dataMap[d.City] = &plateauapi.CityGMLDataset{ - URL: d.CityGML, - Admin: map[string]any{ - "maxlod": d.MaxLOD, - }, - } + dataMap[d.City] = d } - for _, city := range cities { - if _, ok := dataMap[city.ID]; !ok { + for _, city := range all.City { + data := dataMap[city.ID] + if data == nil { continue } - dataMap[city.ID].ID = plateauapi.CityGMLDatasetIDFrom(plateauapi.AreaCode(city.CityCode)) - dataMap[city.ID].Year = city.YearInt() - dataMap[city.ID].RegistrationYear = regYear - dataMap[city.ID].PrefectureCode = plateauapi.AreaCode(plateauapi.AreaCode(city.CityCode).PrefectureCode()) - dataMap[city.ID].PrefectureID = plateauapi.NewID(dataMap[city.ID].PrefectureCode.String(), plateauapi.TypePrefecture) - dataMap[city.ID].CityID = plateauapi.NewID(city.CityCode, plateauapi.TypeCity) - dataMap[city.ID].CityCode = plateauapi.AreaCode(city.CityCode) - dataMap[city.ID].FeatureTypes = all.FeatureTypesOf(city.ID) - dataMap[city.ID].PlateauSpecMinorID = plateauapi.PlateauSpecIDFrom(city.Spec) - dataMap[city.ID].Admin = newAdmin(city.ID, city.SDKStage(), cmsurl, dataMap[city.ID].Admin) + prefCode := plateauapi.AreaCode(city.CityCode).PrefectureCode() + adminExtra := map[string]any{ + "maxlod": []string{data.MaxLOD}, + "citygmlUrl": []string{data.CityGML}, + } + + d := &plateauapi.CityGMLDataset{ + ID: plateauapi.CityGMLDatasetIDFrom(plateauapi.AreaCode(city.CityCode)), + URL: data.CityGML, + Year: city.YearInt(), + RegistrationYear: regYear, + PrefectureCode: plateauapi.AreaCode(prefCode), + PrefectureID: plateauapi.NewID(prefCode, plateauapi.TypePrefecture), + CityID: plateauapi.NewID(city.CityCode, plateauapi.TypeCity), + CityCode: plateauapi.AreaCode(city.CityCode), + FeatureTypes: all.FeatureTypesOf(city.ID), + PlateauSpecMinorID: plateauapi.PlateauSpecIDFrom(city.Spec), + Admin: newAdmin(city.ID, city.SDKStage(), cmsurl, adminExtra), + } + + cityMap[city.ID] = city + res[d.ID] = d + resCity[data.City] = d } - res := make(map[plateauapi.ID]*plateauapi.CityGMLDataset) - for _, v := range dataMap { - res[v.ID] = v + // add citygml urls for sample data + for _, data := range all.Plateau { + for _, d := range data { + if !d.Sample || d.MaxLOD == "" || d.CityGML == "" { + continue + } + + citygml := resCity[d.City] + if citygml == nil { + continue + } + + city := cityMap[d.City] + if city == nil || !city.SDKPublic { + continue + } + + maxlod := citygml.Admin.(map[string]any)["maxlod"].([]string) + citygmlURL := citygml.Admin.(map[string]any)["citygmlUrl"].([]string) + + maxlod = append(maxlod, d.MaxLOD) + citygmlURL = append(citygmlURL, d.CityGML) + + citygml.Admin.(map[string]any)["maxlod"] = maxlod + citygml.Admin.(map[string]any)["citygmlUrl"] = citygmlURL + } } return res, nil diff --git a/server/datacatalog/datacatalogv3/conv_citygml_test.go b/server/datacatalog/datacatalogv3/conv_citygml_test.go new file mode 100644 index 000000000..cea1c7182 --- /dev/null +++ b/server/datacatalog/datacatalogv3/conv_citygml_test.go @@ -0,0 +1,89 @@ +package datacatalogv3 + +import ( + "testing" + + "github.com/eukarya-inc/reearth-plateauview/server/datacatalog/plateauapi" + "github.com/stretchr/testify/assert" +) + +func TestToCityGMLs(t *testing.T) { + regYear := 2023 + + all := &AllData{ + City: []*CityItem{ + { + ID: "city1", + Prefecture: "東京都", + CityName: "東京都23区", + CityNameEn: "tokyo23ku", + CityCode: "13100", + Spec: "第3.3版", + Year: "2023", + SDKPublic: true, + }, + }, + GeospatialjpDataItems: []*GeospatialjpDataItem{ + { + City: "city1", + CityGML: "https://example.com/city1.gml", + MaxLOD: "https://example.com/maxlod1.csv", + }, + }, + Plateau: map[string][]*PlateauFeatureItem{ + "bldg": { + { + City: "city1", + CityGML: "https://example.com/city3.gml", + }, + }, + "ubld": { + { + City: "city1", + Sample: true, + CityGML: "https://example.com/city2.gml", + MaxLOD: "https://example.com/maxlod2.csv", + }, + }, + }, + FeatureTypes: FeatureTypes{ + Plateau: []FeatureType{ + { + Code: "bldg", + }, + { + Code: "ubld", + }, + }, + }, + } + + expected := map[plateauapi.ID]*plateauapi.CityGMLDataset{ + "cg_13100": { + ID: "cg_13100", + Year: 2023, + RegistrationYear: regYear, + URL: "https://example.com/city1.gml", + PrefectureID: "p_13", + PrefectureCode: "13", + CityID: "c_13100", + CityCode: "13100", + PlateauSpecMinorID: "ps_3.3", + FeatureTypes: []string{"bldg", "ubld"}, + Admin: map[string]any{ + "citygmlUrl": []string{ + "https://example.com/city1.gml", + "https://example.com/city2.gml", + }, + "maxlod": []string{ + "https://example.com/maxlod1.csv", + "https://example.com/maxlod2.csv", + }, + }, + }, + } + + res, warnings := toCityGMLs(all, regYear) + assert.Nil(t, warnings) + assert.Equal(t, expected, res) +} diff --git a/server/datacatalog/datacatalogv3/conv_dataset.go b/server/datacatalog/datacatalogv3/conv_dataset.go index 10dda4f64..1561558d5 100644 --- a/server/datacatalog/datacatalogv3/conv_dataset.go +++ b/server/datacatalog/datacatalogv3/conv_dataset.go @@ -152,16 +152,16 @@ func newAdmin(id string, stage stage, cmsurl string, extra any) any { a["stage"] = string(stage) } - if len(a) == 0 { - return nil - } - if extra, ok := extra.(map[string]any); ok && extra != nil { for k, v := range extra { a[k] = v } } + if len(a) == 0 { + return nil + } + return a } diff --git a/server/datacatalog/datacatalogv3/model.go b/server/datacatalog/datacatalogv3/model.go index d1012c08b..a075e4cc4 100644 --- a/server/datacatalog/datacatalogv3/model.go +++ b/server/datacatalog/datacatalogv3/model.go @@ -26,7 +26,7 @@ func (d *AllData) FindPlateauFeatureItemByCityID(ft, cityID string) *PlateauFeat func (all *AllData) FeatureTypesOf(cityID string) (res []string) { for _, ft := range all.FeatureTypes.Plateau { - if dem := all.FindPlateauFeatureItemByCityID(ft.Code, cityID); dem != nil && dem.CityGML != "" { + if p := all.FindPlateauFeatureItemByCityID(ft.Code, cityID); p != nil && p.CityGML != "" { res = append(res, ft.Code) } } diff --git a/server/sdkapi/sdkapiv3/api.go b/server/sdkapi/sdkapiv3/api.go index 3b1760db2..fe5a28337 100644 --- a/server/sdkapi/sdkapiv3/api.go +++ b/server/sdkapi/sdkapiv3/api.go @@ -77,6 +77,9 @@ func (c *APIClient) QueryDatasetFiles(ctx context.Context, id string) (DatasetFi defer resp.Body.Close() if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusNotFound { + return nil, nil + } return nil, fmt.Errorf("error response: %s", resp.Status) }