Add RestCatalog create_namespace

.formatter.exs

@@ -0,0 +1,4 @@
+# Used by "mix format"
+[
+  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
+]

.github/workflows/ci.yml

@@ -0,0 +1,55 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+env:
+  otp: "25.0"
+  elixir: "1.16.2"
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Install Erlang & Elixir
+      uses: erlef/setup-beam@v1
+      with:
+        otp-version: ${{ env.otp }}
+        elixir-version: ${{ env.elixir }}
+
+    - uses: hoverkraft-tech/compose-action@v2.0.0
+      with:
+        compose-file: "./docker-compose.yml"
+        down-flags: "--remove-orphans"
+        up-flags: "--no-start"
+
+    - name: Install dependencies
+      run: mix deps.get
+
+    - name: Check formatting
+      run: mix format --check-formatted
+
+    - name: Check unused deps
+      run: mix deps.unlock --check-unused
+
+    - name: Check warnings
+      run: mix compile --warnings-as-errors
+
+    - name: Run tests
+      run: mix test --exclude integration
+
+    - name: Run integration tests
+      run: |
+        docker-compose up -d
+        mix test --only integration
+        docker-compose down --remove-orphans

.gitignore

@@ -0,0 +1,26 @@
+# The directory Mix will write compiled artifacts to.
+/_build/
+
+# If you run "mix test --cover", coverage assets end up here.
+/cover/
+
+# The directory Mix downloads your dependencies sources to.
+/deps/
+
+# Where third-party dependencies like ExDoc output generated docs.
+/doc/
+
+# Ignore .fetch files in case you like to edit your project deps locally.
+/.fetch
+
+# If the VM crashes, it generates a dump, let's ignore it too.
+erl_crash.dump
+
+# Also ignore archive artifacts (built via "mix archive.build").
+*.ez
+
+# Ignore package tarball (built via "mix hex.build").
+ex_iceberg-*.tar
+
+# Temporary files, for example, from tests.
+/tmp/

.tool-versions

@@ -0,0 +1,2 @@
+elixir 1.16.2-otp-25
+erlang 25.3.2.10

docker-compose.yml

@@ -0,0 +1,49 @@
+version: '3.8'
+
+services:
+  rest:
+    image: tabulario/iceberg-rest:0.10.0
+    environment:
+      - AWS_ACCESS_KEY_ID=admin
+      - AWS_SECRET_ACCESS_KEY=password
+      - AWS_REGION=us-east-1
+      - CATALOG_CATALOG__IMPL=org.apache.iceberg.jdbc.JdbcCatalog
+      - CATALOG_URI=jdbc:sqlite:file:/tmp/iceberg_rest?mode=memory
+      - CATALOG_WAREHOUSE=s3://icebergdata/demo
+      - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
+      - CATALOG_S3_ENDPOINT=http://minio:9000
+    depends_on:
+      - minio
+    links:
+      - minio:icebergdata.minio
+    ports:
+      - "8181:8181"
+
+  minio:
+    image: minio/minio:RELEASE.2024-03-07T00-43-48Z
+    environment:
+      - MINIO_ROOT_USER=admin
+      - MINIO_ROOT_PASSWORD=password
+      - MINIO_DOMAIN=minio
+    expose:
+      - 9001
+      - 9000
+    command: [ "server", "/data", "--console-address", ":9001" ]
+
+  mc:
+    depends_on:
+      - minio
+    image: minio/mc:RELEASE.2024-03-07T00-31-49Z
+    environment:
+      - AWS_ACCESS_KEY_ID=admin
+      - AWS_SECRET_ACCESS_KEY=password
+      - AWS_REGION=us-east-1
+    entrypoint: >
+      /bin/sh -c "
+        until (/usr/bin/mc config host add minio http://minio:9000 admin password) do
+          echo '...waiting...' && sleep 1;
+        done;
+        /usr/bin/mc mb --ignore-existing minio/icebergdata;
+        /usr/bin/mc policy set public minio/icebergdata;
+        tail -f /dev/null
+      "

lib/ex_iceberg.ex

@@ -0,0 +1,2 @@
+defmodule ExIceberg do
+end

lib/ex_iceberg/catalog.ex

@@ -0,0 +1,8 @@
+defmodule ExIceberg.Catalog do
+  @callback create_namespace(t(), String.t(), map()) :: :ok | {:error, String.t()}
+
+  @type t :: %{
+          name: String.t(),
+          properties: map()
+        }
+end

lib/ex_iceberg/catalog/rest_catalog.ex

@@ -0,0 +1,77 @@
+defmodule ExIceberg.Catalog.RestCatalog do
+  @moduledoc """
+  Module to interact with the REST catalog of Apache Iceberg.
+  """
+
+  @behaviour ExIceberg.Catalog
+
+  defstruct name: nil, properties: %{}, http_client: ExIceberg.ReqClient
+
+  @type t :: %__MODULE__{
+          name: String.t(),
+          properties: map(),
+          http_client: module()
+        }
+
+  @doc """
+  Initializes a new RestCatalog.
+
+  ## Parameters
+
+    - `name`: The name of the catalog
+    - `properties`: A map of properties for the catalog
+
+  ## Examples
+
+      iex> ExIceberg.Catalog.RestCatalog.new("my_catalog", %{"uri" => "http://localhost:8181"})
+      %ExIceberg.Catalog.RestCatalog{
+        name: "my_catalog",
+        properties: %{"uri" => "http://localhost:8181"},
+        http_client: ExIceberg.ReqClient
+      }
+  """
+  def new(name, properties, http_client \\ ExIceberg.ReqClient) do
+    %__MODULE__{
+      name: name,
+      properties: properties,
+      http_client: http_client
+    }
+  end
+
+  @doc """
+  Creates a new namespace in the catalog.
+
+  ## Parameters
+
+    - `catalog`: The catalog struct
+    - `namespace`: The name of the namespace
+    - `properties`: A map of properties for the namespace
+
+  ## Examples
+
+      iex> catalog = ExIceberg.Catalog.RestCatalog.new("my_catalog", %{"uri" => "http://localhost:8181"})
+      iex> ExIceberg.Catalog.RestCatalog.create_namespace(catalog, "new_namespace", %{"property" => "value"})
+      :ok
+  """
+  @impl true
+  def create_namespace(
+        %__MODULE__{properties: %{"uri" => uri}, http_client: http_client} = _catalog,
+        namespace,
+        properties
+      ) do
+    url = uri <> "/v1/namespaces"
+    body = %{"namespace" => [namespace], "properties" => properties}
+    headers = [{"Content-Type", "application/json"}]
+
+    case http_client.request(:post, url, body, headers) do
+      {:ok, _body} ->
+        :ok
+
+      {:error, %Req.Response{status: status, body: %{"error" => %{"message" => message}}}} ->
+        {:error, "Request failed with status #{status}: #{message}"}
+
+      {:error, %Req.TransportError{reason: reason}} ->
+        {:error, "HTTP request failed: #{inspect(reason)}"}
+    end
+  end
+end

lib/ex_iceberg/http_client.ex

@@ -0,0 +1,3 @@
+defmodule ExIceberg.HTTPClient do
+  @callback request(atom(), String.t(), map(), list()) :: {:ok, any()} | {:error, any()}
+end

lib/ex_iceberg/req_client.ex

@@ -0,0 +1,24 @@
+defmodule ExIceberg.ReqClient do
+  @behaviour ExIceberg.HTTPClient
+
+  def request(method, url, body, headers) do
+    req =
+      Req.new(
+        method: method,
+        url: url,
+        body: Jason.encode!(body),
+        headers: headers
+      )
+
+    case Req.request(req) do
+      {:ok, %Req.Response{status: status, body: body}} when status in 200..299 ->
+        {:ok, body}
+
+      {:ok, %Req.Response{status: status, body: body}} ->
+        {:error, %Req.Response{status: status, body: body}}
+
+      {:error, %Req.TransportError{reason: reason}} ->
+        {:error, %Req.TransportError{reason: reason}}
+    end
+  end
+end

mix.exs

@@ -0,0 +1,54 @@
+defmodule ExIceberg.MixProject do
+  use Mix.Project
+
+  @version "0.1.0"
+  @description "ExIceberg is an Elixir library for interacting with Apache Iceberg."
+
+  def project do
+    [
+      app: :ex_iceberg,
+      name: "ExIceberg",
+      version: @version,
+      description: @description,
+      elixir: "~> 1.16",
+      start_permanent: Mix.env() == :prod,
+      preferred_cli_env: [
+        "test.all": :test,
+        "hex.publish": :docs
+      ],
+      deps: deps(),
+      aliases: aliases(),
+      package: package()
+    ]
+  end
+
+  def application do
+    [
+      extra_applications: [:logger]
+    ]
+  end
+
+  def aliases do
+    [
+      "test.all": ["test --include integration"]
+    ]
+  end
+
+  defp deps do
+    [
+      {:req, "~> 0.5"},
+      {:jason, "~> 1.2"},
+      {:mox, "~> 1.0", only: :test},
+      {:ex_doc, "~> 0.34", only: :dev, runtime: false}
+    ]
+  end
+
+  defp package do
+    [
+      licenses: ["Apache-2.0"],
+      links: %{
+        "GitHub" => "https://github.com/ndrluis/ex_iceberg"
+      }
+    ]
+  end
+end

mix.lock

@@ -0,0 +1,19 @@
+%{
+  "castore": {:hex, :castore, "1.0.7", "b651241514e5f6956028147fe6637f7ac13802537e895a724f90bf3e36ddd1dd", [:mix], [], "hexpm", "da7785a4b0d2a021cd1292a60875a784b6caef71e76bf4917bdee1f390455cf5"},
+  "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"},
+  "ex_doc": {:hex, :ex_doc, "0.34.0", "ab95e0775db3df71d30cf8d78728dd9261c355c81382bcd4cefdc74610bef13e", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "60734fb4c1353f270c3286df4a0d51e65a2c1d9fba66af3940847cc65a8066d7"},
+  "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"},
+  "hpax": {:hex, :hpax, "0.2.0", "5a58219adcb75977b2edce5eb22051de9362f08236220c9e859a47111c194ff5", [:mix], [], "hexpm", "bea06558cdae85bed075e6c036993d43cd54d447f76d8190a8db0dc5893fa2f1"},
+  "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
+  "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"},
+  "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"},
+  "makeup_erlang": {:hex, :makeup_erlang, "1.0.0", "6f0eff9c9c489f26b69b61440bf1b238d95badae49adac77973cbacae87e3c2e", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "ea7a9307de9d1548d2a72d299058d1fd2339e3d398560a0e46c27dab4891e4d2"},
+  "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"},
+  "mint": {:hex, :mint, "1.6.0", "88a4f91cd690508a04ff1c3e28952f322528934be541844d54e0ceb765f01d5e", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "3c5ae85d90a5aca0a49c0d8b67360bbe407f3b54f1030a111047ff988e8fefaa"},
+  "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"},
+  "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"},
+  "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"},
+  "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"},
+  "req": {:hex, :req, "0.5.0", "6d8a77c25cfc03e06a439fb12ffb51beade53e3fe0e2c5e362899a18b50298b3", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "dda04878c1396eebbfdec6db6f3d4ca609e5c8846b7ee88cc56eb9891406f7a3"},
+  "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"},
+}

test/integration/catalog/rest_catalog_integration_test.exs

@@ -0,0 +1,29 @@
+defmodule ExIceberg.Catalog.RestCatalogIntegrationTest do
+  use ExUnit.Case, async: true
+  alias ExIceberg.Catalog.RestCatalog
+
+  @moduletag :integration
+
+  @test_uri "http://localhost:8181"
+
+  defp generate_unique_name(base) do
+    hash = :crypto.strong_rand_bytes(8) |> Base.encode16() |> String.downcase()
+    "#{base}_#{hash}"
+  end
+
+  test "successfully creates a new namespace" do
+    namespace = generate_unique_name("some_namespace")
+    catalog = RestCatalog.new("my_catalog", %{"uri" => @test_uri})
+    assert :ok == RestCatalog.create_namespace(catalog, namespace, %{})
+  end
+
+  test "fails to create a new namespace due to conflict" do
+    namespace = generate_unique_name("examples")
+    catalog = RestCatalog.new("my_catalog", %{"uri" => @test_uri})
+
+    assert :ok == RestCatalog.create_namespace(catalog, namespace, %{})
+
+    assert {:error, "Request failed with status 409: Namespace already exists: #{namespace}"} ==
+             RestCatalog.create_namespace(catalog, namespace, %{})
+  end
+end

test/test_helper.exs

@@ -0,0 +1,3 @@
+ExUnit.start(exclude: [:integration])
+Mox.defmock(ExIceberg.MockHTTPClient, for: ExIceberg.HTTPClient)
+Application.ensure_all_started(:mox)