Add insecure option, include bucket name in AWS credentials, best-eff…

…ort+zoom flag (#18)

* implement minZ maxZ logic

* add tiles_c

* revert image.go changes

* actually loop sources backwards and reimplement opaque check

* actually loop sources backwards and reimplement opaque check

* insecure https

* add insecure option

* include bucketname in AWS credentials env variables

* fix zoom level mismatch skip in combination with bestEffort flag

Co-authored-by: Alexander Waldenmaier <alexander.waldenmaier@ororatech.com>

S3Backend/s3backend.go

@@ -3,10 +3,11 @@ package S3Backend
 import (
 	"bytes"
 	"context"
-	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
 	"os"
 	"strings"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
 )
 
 type S3Backend struct {
@@ -15,12 +16,14 @@ type S3Backend struct {
 	BasePath string
 }
 
-func NewS3Backend(path string) (*S3Backend, error) {
+func NewS3Backend(path string, insecure bool) (*S3Backend, error) {
 	pathComponents := strings.Split(path[5:], "/")
 
+	accessKey := os.Getenv(pathComponents[0] + "_" + pathComponents[1] + "_ACCESS_KEY_ID")
+	secretKey := os.Getenv(pathComponents[0] + "_" + pathComponents[1] + "_SECRET_ACCESS_KEY")
 	minioClient, err := minio.New(pathComponents[0], &minio.Options{
-		Creds:  credentials.NewStaticV4(os.Getenv(pathComponents[0]+"_ACCESS_KEY_ID"), os.Getenv(pathComponents[0]+"_SECRET_ACCESS_KEY"), ""),
-		Secure: true,
+		Creds:  credentials.NewStaticV4(accessKey, secretKey, ""),
+		Secure: !insecure,
 	})
 
 	if err != nil {

main.go

@@ -42,13 +42,16 @@ func atomicAverage(target *int64, channel *chan time.Duration) {
 	}
 }
 
+var insecure bool
+
 func main() {
 	numWorkers := flag.Int("parallel", 2, "Number of parallel threads to use for processing")
 	quiet := flag.Bool("quiet", false, "Don't output progress information")
 	debug := flag.Bool("debug", false, "Enable debugging (tracing and some perf counters)")
 	report := flag.Bool("report", false, "Enable periodic reports (every min); intended for non-interactive environments")
 	bestEffort := flag.Bool("best-effort", false, "Best-effort merging: ignore erroneous tilesets completely and silently skip single failed tiles.")
 	zoom := flag.String("zoom", "", "Restrict/manually set zoom levels to work on, in the form of 'minZ-maxZ' (e.g. '1-8'). If this option is specified, prioritile does not try to automatically detect the zoom levels of the target but rather uses these hardcoded ones.")
+	insecure = *flag.Bool("insecure", false, "Do not use https for remote sources/target")
 	flag.Usage = func() {
 		fmt.Fprintln(os.Stderr, "Usage: prioritile [-zoom '1-8'] [-debug] [-report] [-best-effort] [-parallel=2] /tiles/target/ /tiles/source1/ [s3://foo/tiles/source2/ [...]]")
 		fmt.Fprintln(os.Stderr, "")
@@ -63,9 +66,9 @@ func main() {
 		fmt.Fprintln(os.Stderr, "- Resolution of corresponding tiles matches")
 		fmt.Fprintln(os.Stderr, "")
 		fmt.Fprintln(os.Stderr, "S3 disk backends are supported as source and target by prefixing the tile")
-		fmt.Fprintln(os.Stderr, "directories with 's3://', e.g. 's3://example.com/source/'.")
-		fmt.Fprintln(os.Stderr, "S3 authentication information is read from environment variables prefixed with the target hostname:")
-		fmt.Fprintln(os.Stderr, "example.com_ACCESS_KEY_ID, example.com_SECRET_ACCESS_KEY")
+		fmt.Fprintln(os.Stderr, "directories with 's3://', e.g. 's3://example.com/foobucket/'.")
+		fmt.Fprintln(os.Stderr, "S3 authentication information is read from environment variables prefixed with the target hostname and bucketname:")
+		fmt.Fprintln(os.Stderr, "example.com_foobucket_ACCESS_KEY_ID, example.com_foobucket_SECRET_ACCESS_KEY")
 		fmt.Fprintln(os.Stderr, "")
 		flag.PrintDefaults()
 	}
@@ -111,7 +114,7 @@ func main() {
 		}
 	}
 
-	sources, errs := discoverTilesets(flag.Args()[1:], target.MinZ, target.MaxZ)
+	sources, errs := discoverTilesets(flag.Args()[1:], target, *bestEffort)
 	if errs != nil && !*bestEffort {
 		log.Fatalf("could not discover tilesets: %v", errs)
 	}
@@ -312,7 +315,7 @@ func main() {
 
 func stringToBackend(pathSpec string) (StorageBackend, error) {
 	if strings.HasPrefix(pathSpec, "s3://") {
-		backend, err := S3Backend.NewS3Backend(pathSpec)
+		backend, err := S3Backend.NewS3Backend(pathSpec, insecure)
 		if err != nil {
 			return nil, err
 		}

tileset.go

@@ -17,29 +17,29 @@ func (t TilesetDescriptor) String() string {
 	return fmt.Sprintf("%d-%d", t.MaxZ, t.MinZ)
 }
 
-func discoverTilesets(paths []string, minZ int, maxZ int) ([]TilesetDescriptor, []error) {
+func discoverTilesets(paths []string, target TilesetDescriptor, bestEffort bool) ([]TilesetDescriptor, []error) {
 	var tilesets []TilesetDescriptor
 	var errors []error
 
-	// XXX if discovery for the target tileset fails, the first source might be used as a target, which is somewhat
-	// undesirable, I believe
 	for _, path := range paths {
 		backend, err := stringToBackend(path)
 		if err != nil {
 			errors = append(errors, err)
 			continue
 		}
 
-		tileset, err := discoverTileset(backend, minZ, maxZ)
+		tileset, err := discoverTileset(backend, target.MinZ, target.MaxZ)
 
 		if err != nil {
 			errors = append(errors, fmt.Errorf("could not discover tileset: %v in %s", err, path))
 			continue
 		}
 
-		if len(tilesets) > 0 && (tilesets[0].MaxZ != tileset.MaxZ || tilesets[0].MinZ != tileset.MinZ) {
+		if len(tilesets) > 0 && (target.MaxZ != tileset.MaxZ || target.MinZ != tileset.MinZ) {
 			errors = append(errors, fmt.Errorf("zoom level mismatch for target and source %s", path))
-			continue
+			if !bestEffort {
+				continue
+			}
 		}
 		tilesets = append(tilesets, tileset)
 	}