From 3aa4e16bab539a27d17c1ffcf844be0df359f2e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andre=20Anast=C3=A1cio?= Date: Sat, 8 Jun 2024 18:15:18 -0300 Subject: [PATCH] Add RestCatalog create_namespace --- .formatter.exs | 4 + .github/workflows/ci.yml | 55 +++++++++++++ .gitignore | 26 +++++++ .tool-versions | 2 + docker-compose.yml | 49 ++++++++++++ lib/ex_iceberg.ex | 2 + lib/ex_iceberg/catalog.ex | 8 ++ lib/ex_iceberg/catalog/rest_catalog.ex | 77 +++++++++++++++++++ lib/ex_iceberg/http_client.ex | 3 + lib/ex_iceberg/req_client.ex | 24 ++++++ mix.exs | 54 +++++++++++++ mix.lock | 19 +++++ .../catalog/rest_catalog_integration_test.exs | 29 +++++++ test/test_helper.exs | 3 + test/unit/catalog/rest_catalog_test.exs | 52 +++++++++++++ 15 files changed, 407 insertions(+) create mode 100644 .formatter.exs create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 .tool-versions create mode 100644 docker-compose.yml create mode 100644 lib/ex_iceberg.ex create mode 100644 lib/ex_iceberg/catalog.ex create mode 100644 lib/ex_iceberg/catalog/rest_catalog.ex create mode 100644 lib/ex_iceberg/http_client.ex create mode 100644 lib/ex_iceberg/req_client.ex create mode 100644 mix.exs create mode 100644 mix.lock create mode 100644 test/integration/catalog/rest_catalog_integration_test.exs create mode 100644 test/test_helper.exs create mode 100644 test/unit/catalog/rest_catalog_test.exs diff --git a/.formatter.exs b/.formatter.exs new file mode 100644 index 0000000..d2cda26 --- /dev/null +++ b/.formatter.exs @@ -0,0 +1,4 @@ +# Used by "mix format" +[ + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] +] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2653bad --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,55 @@ +name: CI + +on: + push: + branches: + - main + pull_request: + branches: + - main + +env: + otp: "25.0" + elixir: "1.16.2" + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install Erlang & Elixir + uses: erlef/setup-beam@v1 + with: + otp-version: ${{ env.otp }} + elixir-version: ${{ env.elixir }} + + - uses: hoverkraft-tech/compose-action@v2.0.0 + with: + compose-file: "./docker-compose.yml" + down-flags: "--remove-orphans" + up-flags: "--no-start" + + - name: Install dependencies + run: mix deps.get + + - name: Check formatting + run: mix format --check-formatted + + - name: Check unused deps + run: mix deps.unlock --check-unused + + - name: Check warnings + run: mix compile --warnings-as-errors + + - name: Run tests + run: mix test --exclude integration + + - name: Run integration tests + run: | + docker-compose up -d + mix test --only integration + docker-compose down --remove-orphans diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..07c5e67 --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +# The directory Mix will write compiled artifacts to. +/_build/ + +# If you run "mix test --cover", coverage assets end up here. +/cover/ + +# The directory Mix downloads your dependencies sources to. +/deps/ + +# Where third-party dependencies like ExDoc output generated docs. +/doc/ + +# Ignore .fetch files in case you like to edit your project deps locally. +/.fetch + +# If the VM crashes, it generates a dump, let's ignore it too. +erl_crash.dump + +# Also ignore archive artifacts (built via "mix archive.build"). +*.ez + +# Ignore package tarball (built via "mix hex.build"). +ex_iceberg-*.tar + +# Temporary files, for example, from tests. +/tmp/ diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 0000000..ef26c8a --- /dev/null +++ b/.tool-versions @@ -0,0 +1,2 @@ +elixir 1.16.2-otp-25 +erlang 25.3.2.10 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..26db6ab --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,49 @@ +version: '3.8' + +services: + rest: + image: tabulario/iceberg-rest:0.10.0 + environment: + - AWS_ACCESS_KEY_ID=admin + - AWS_SECRET_ACCESS_KEY=password + - AWS_REGION=us-east-1 + - CATALOG_CATALOG__IMPL=org.apache.iceberg.jdbc.JdbcCatalog + - CATALOG_URI=jdbc:sqlite:file:/tmp/iceberg_rest?mode=memory + - CATALOG_WAREHOUSE=s3://icebergdata/demo + - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO + - CATALOG_S3_ENDPOINT=http://minio:9000 + depends_on: + - minio + links: + - minio:icebergdata.minio + ports: + - "8181:8181" + + minio: + image: minio/minio:RELEASE.2024-03-07T00-43-48Z + environment: + - MINIO_ROOT_USER=admin + - MINIO_ROOT_PASSWORD=password + - MINIO_DOMAIN=minio + expose: + - 9001 + - 9000 + command: [ "server", "/data", "--console-address", ":9001" ] + + mc: + depends_on: + - minio + image: minio/mc:RELEASE.2024-03-07T00-31-49Z + environment: + - AWS_ACCESS_KEY_ID=admin + - AWS_SECRET_ACCESS_KEY=password + - AWS_REGION=us-east-1 + entrypoint: > + /bin/sh -c " + until (/usr/bin/mc config host add minio http://minio:9000 admin password) do + echo '...waiting...' && sleep 1; + done; + /usr/bin/mc mb --ignore-existing minio/icebergdata; + /usr/bin/mc policy set public minio/icebergdata; + tail -f /dev/null + " diff --git a/lib/ex_iceberg.ex b/lib/ex_iceberg.ex new file mode 100644 index 0000000..a2a076a --- /dev/null +++ b/lib/ex_iceberg.ex @@ -0,0 +1,2 @@ +defmodule ExIceberg do +end diff --git a/lib/ex_iceberg/catalog.ex b/lib/ex_iceberg/catalog.ex new file mode 100644 index 0000000..ccc400d --- /dev/null +++ b/lib/ex_iceberg/catalog.ex @@ -0,0 +1,8 @@ +defmodule ExIceberg.Catalog do + @callback create_namespace(t(), String.t(), map()) :: :ok | {:error, String.t()} + + @type t :: %{ + name: String.t(), + properties: map() + } +end diff --git a/lib/ex_iceberg/catalog/rest_catalog.ex b/lib/ex_iceberg/catalog/rest_catalog.ex new file mode 100644 index 0000000..a7c7d9c --- /dev/null +++ b/lib/ex_iceberg/catalog/rest_catalog.ex @@ -0,0 +1,77 @@ +defmodule ExIceberg.Catalog.RestCatalog do + @moduledoc """ + Module to interact with the REST catalog of Apache Iceberg. + """ + + @behaviour ExIceberg.Catalog + + defstruct name: nil, properties: %{}, http_client: ExIceberg.ReqClient + + @type t :: %__MODULE__{ + name: String.t(), + properties: map(), + http_client: module() + } + + @doc """ + Initializes a new RestCatalog. + + ## Parameters + + - `name`: The name of the catalog + - `properties`: A map of properties for the catalog + + ## Examples + + iex> ExIceberg.Catalog.RestCatalog.new("my_catalog", %{"uri" => "http://localhost:8181"}) + %ExIceberg.Catalog.RestCatalog{ + name: "my_catalog", + properties: %{"uri" => "http://localhost:8181"}, + http_client: ExIceberg.ReqClient + } + """ + def new(name, properties, http_client \\ ExIceberg.ReqClient) do + %__MODULE__{ + name: name, + properties: properties, + http_client: http_client + } + end + + @doc """ + Creates a new namespace in the catalog. + + ## Parameters + + - `catalog`: The catalog struct + - `namespace`: The name of the namespace + - `properties`: A map of properties for the namespace + + ## Examples + + iex> catalog = ExIceberg.Catalog.RestCatalog.new("my_catalog", %{"uri" => "http://localhost:8181"}) + iex> ExIceberg.Catalog.RestCatalog.create_namespace(catalog, "new_namespace", %{"property" => "value"}) + :ok + """ + @impl true + def create_namespace( + %__MODULE__{properties: %{"uri" => uri}, http_client: http_client} = _catalog, + namespace, + properties + ) do + url = uri <> "/v1/namespaces" + body = %{"namespace" => [namespace], "properties" => properties} + headers = [{"Content-Type", "application/json"}] + + case http_client.request(:post, url, body, headers) do + {:ok, _body} -> + :ok + + {:error, %Req.Response{status: status, body: %{"error" => %{"message" => message}}}} -> + {:error, "Request failed with status #{status}: #{message}"} + + {:error, %Req.TransportError{reason: reason}} -> + {:error, "HTTP request failed: #{inspect(reason)}"} + end + end +end diff --git a/lib/ex_iceberg/http_client.ex b/lib/ex_iceberg/http_client.ex new file mode 100644 index 0000000..e4f8ed3 --- /dev/null +++ b/lib/ex_iceberg/http_client.ex @@ -0,0 +1,3 @@ +defmodule ExIceberg.HTTPClient do + @callback request(atom(), String.t(), map(), list()) :: {:ok, any()} | {:error, any()} +end diff --git a/lib/ex_iceberg/req_client.ex b/lib/ex_iceberg/req_client.ex new file mode 100644 index 0000000..99f70cf --- /dev/null +++ b/lib/ex_iceberg/req_client.ex @@ -0,0 +1,24 @@ +defmodule ExIceberg.ReqClient do + @behaviour ExIceberg.HTTPClient + + def request(method, url, body, headers) do + req = + Req.new( + method: method, + url: url, + body: Jason.encode!(body), + headers: headers + ) + + case Req.request(req) do + {:ok, %Req.Response{status: status, body: body}} when status in 200..299 -> + {:ok, body} + + {:ok, %Req.Response{status: status, body: body}} -> + {:error, %Req.Response{status: status, body: body}} + + {:error, %Req.TransportError{reason: reason}} -> + {:error, %Req.TransportError{reason: reason}} + end + end +end diff --git a/mix.exs b/mix.exs new file mode 100644 index 0000000..f2df7e9 --- /dev/null +++ b/mix.exs @@ -0,0 +1,54 @@ +defmodule ExIceberg.MixProject do + use Mix.Project + + @version "0.1.0" + @description "ExIceberg is an Elixir library for interacting with Apache Iceberg." + + def project do + [ + app: :ex_iceberg, + name: "ExIceberg", + version: @version, + description: @description, + elixir: "~> 1.16", + start_permanent: Mix.env() == :prod, + preferred_cli_env: [ + "test.all": :test, + "hex.publish": :docs + ], + deps: deps(), + aliases: aliases(), + package: package() + ] + end + + def application do + [ + extra_applications: [:logger] + ] + end + + def aliases do + [ + "test.all": ["test --include integration"] + ] + end + + defp deps do + [ + {:req, "~> 0.5"}, + {:jason, "~> 1.2"}, + {:mox, "~> 1.0", only: :test}, + {:ex_doc, "~> 0.34", only: :dev, runtime: false} + ] + end + + defp package do + [ + licenses: ["Apache-2.0"], + links: %{ + "GitHub" => "https://github.com/ndrluis/ex_iceberg" + } + ] + end +end diff --git a/mix.lock b/mix.lock new file mode 100644 index 0000000..d2f6590 --- /dev/null +++ b/mix.lock @@ -0,0 +1,19 @@ +%{ + "castore": {:hex, :castore, "1.0.7", "b651241514e5f6956028147fe6637f7ac13802537e895a724f90bf3e36ddd1dd", [:mix], [], "hexpm", "da7785a4b0d2a021cd1292a60875a784b6caef71e76bf4917bdee1f390455cf5"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"}, + "ex_doc": {:hex, :ex_doc, "0.34.0", "ab95e0775db3df71d30cf8d78728dd9261c355c81382bcd4cefdc74610bef13e", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "60734fb4c1353f270c3286df4a0d51e65a2c1d9fba66af3940847cc65a8066d7"}, + "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"}, + "hpax": {:hex, :hpax, "0.2.0", "5a58219adcb75977b2edce5eb22051de9362f08236220c9e859a47111c194ff5", [:mix], [], "hexpm", "bea06558cdae85bed075e6c036993d43cd54d447f76d8190a8db0dc5893fa2f1"}, + "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, + "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"}, + "makeup_erlang": {:hex, :makeup_erlang, "1.0.0", "6f0eff9c9c489f26b69b61440bf1b238d95badae49adac77973cbacae87e3c2e", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "ea7a9307de9d1548d2a72d299058d1fd2339e3d398560a0e46c27dab4891e4d2"}, + "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"}, + "mint": {:hex, :mint, "1.6.0", "88a4f91cd690508a04ff1c3e28952f322528934be541844d54e0ceb765f01d5e", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "3c5ae85d90a5aca0a49c0d8b67360bbe407f3b54f1030a111047ff988e8fefaa"}, + "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"}, + "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, + "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, + "req": {:hex, :req, "0.5.0", "6d8a77c25cfc03e06a439fb12ffb51beade53e3fe0e2c5e362899a18b50298b3", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "dda04878c1396eebbfdec6db6f3d4ca609e5c8846b7ee88cc56eb9891406f7a3"}, + "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, +} diff --git a/test/integration/catalog/rest_catalog_integration_test.exs b/test/integration/catalog/rest_catalog_integration_test.exs new file mode 100644 index 0000000..e00230a --- /dev/null +++ b/test/integration/catalog/rest_catalog_integration_test.exs @@ -0,0 +1,29 @@ +defmodule ExIceberg.Catalog.RestCatalogIntegrationTest do + use ExUnit.Case, async: true + alias ExIceberg.Catalog.RestCatalog + + @moduletag :integration + + @test_uri "http://localhost:8181" + + defp generate_unique_name(base) do + hash = :crypto.strong_rand_bytes(8) |> Base.encode16() |> String.downcase() + "#{base}_#{hash}" + end + + test "successfully creates a new namespace" do + namespace = generate_unique_name("some_namespace") + catalog = RestCatalog.new("my_catalog", %{"uri" => @test_uri}) + assert :ok == RestCatalog.create_namespace(catalog, namespace, %{}) + end + + test "fails to create a new namespace due to conflict" do + namespace = generate_unique_name("examples") + catalog = RestCatalog.new("my_catalog", %{"uri" => @test_uri}) + + assert :ok == RestCatalog.create_namespace(catalog, namespace, %{}) + + assert {:error, "Request failed with status 409: Namespace already exists: #{namespace}"} == + RestCatalog.create_namespace(catalog, namespace, %{}) + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs new file mode 100644 index 0000000..a46bd89 --- /dev/null +++ b/test/test_helper.exs @@ -0,0 +1,3 @@ +ExUnit.start(exclude: [:integration]) +Mox.defmock(ExIceberg.MockHTTPClient, for: ExIceberg.HTTPClient) +Application.ensure_all_started(:mox) diff --git a/test/unit/catalog/rest_catalog_test.exs b/test/unit/catalog/rest_catalog_test.exs new file mode 100644 index 0000000..38ec4e2 --- /dev/null +++ b/test/unit/catalog/rest_catalog_test.exs @@ -0,0 +1,52 @@ +defmodule ExIceberg.Catalog.RestCatalogTest do + use ExUnit.Case, async: true + import Mox + alias ExIceberg.Catalog.RestCatalog + + setup :verify_on_exit! + + @test_uri "http://localhost:8181" + @headers [{"Content-Type", "application/json"}] + + describe "create_namespace/3" do + test "successfully creates a new namespace" do + ExIceberg.MockHTTPClient + |> expect(:request, fn :post, + @test_uri <> "/v1/namespaces", + %{"namespace" => ["some_namespace"], "properties" => %{}}, + @headers -> + {:ok, %Req.Response{status: 200, body: %{}}} + end) + + catalog = RestCatalog.new("my_catalog", %{"uri" => @test_uri}, ExIceberg.MockHTTPClient) + assert :ok == RestCatalog.create_namespace(catalog, "some_namespace", %{}) + end + + test "fails to create a new namespace due to conflict" do + ExIceberg.MockHTTPClient + |> expect(:request, fn :post, + @test_uri <> "/v1/namespaces", + %{"namespace" => ["examples"], "properties" => %{}}, + @headers -> + {:error, + %Req.Response{ + status: 409, + body: %{ + "error" => %{ + "message" => + "Namespace already exists: examples in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", + "type" => "AlreadyExistsException", + "code" => 409 + } + } + }} + end) + + catalog = RestCatalog.new("my_catalog", %{"uri" => @test_uri}, ExIceberg.MockHTTPClient) + + assert {:error, + "Request failed with status 409: Namespace already exists: examples in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e"} == + RestCatalog.create_namespace(catalog, "examples", %{}) + end + end +end