Feature/schema namespace map (#208)

* add schema_namespace_map config option

* implement schema_namespace_map for when generate_namespace_folders is enabled

- Use entire namespace for both module definition and folder structure
- (zeitwerk fix): use camelize instead of classiify to allow plural module names
- (zeitwerk fix): don't use singularize to allow plural directory names
- check schema_namespace_map for largest matching namespace reduction and apply it

* regenerate test schema classes, add schema for testing

* add spec, update snapshots

* changelog

* update docs

* rubocop

* use keys array for lookup

* update docs to specify base module will be modified and that the keys are prefixes

* update module_namespace assignment & tweak regex to match `.` character rather than any

* update comment

* improve directories assignment

* rename generate_namespace_folders to use_full_namespace

* fix spec

CHANGELOG.md

@@ -24,6 +24,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Feature: `should_consume?` method accepts BatchRecord associations
 - Feature: Reintroduce `filter_records` for bulk filtering of records prior to insertion
 - Feature: Return valid and invalid records saved during consumption for further processing in `batch_consumption.valid_records` and `batch_consumption.invalid_records` ActiveSupport Notifications
+- Feature: Rename configuration option `generate_namespace_folders` to `use_full_namespace`. This will now use the entire namespace when generating schema classes
+- Feature: Add configuration option `schema_namespace_map` to enable full control over the namespace for generated schema classes. Requires `use_full_namespace`
 
 # 1.22.5 - 2023-07-18
 - Fix: Fixed buffer overflow crash with DB producer.

docs/CONFIGURATION.md

@@ -210,7 +210,8 @@ schema.path|nil|Local path to find your schemas.
 schema.use_schema_classes|false|Set this to true to use generated schema classes in your application.
 schema.generated_class_path|`app/lib/schema_classes`|Local path to generated schema classes.
 schema.nest_child_schemas|false|Set to true to nest subschemas within the generated class for the parent schema.
-schema.generate_namespace_folders|false|Set to true to generate folders for schemas matching the last part of the namespace.
+schema.use_full_namespace|false|Set to true to generate folders for schemas matching the full namespace.
+schema.schema_namespace_map|{}|A map of namespace prefixes to base module name(s). Example: { 'com.mycompany.suborg' => ['SchemaClasses'] }. Requires `use_full_namespace` to be true.
 
 ## Database Producer Configuration
 

lib/deimos/config/configuration.rb

@@ -380,7 +380,13 @@ def self.configure_producer_or_consumer(kafka_config)
 
       # Set to true to generate folders matching the last part of the schema namespace.
       # @return [Boolean]
-      setting :generate_namespace_folders, false
+      setting :use_full_namespace, false
+
+      # Use this option to reduce nesting when using use_full_namespace.
+      # For example: { 'com.mycompany.suborg' => 'SchemaClasses' }
+      # would replace a prefixed with the given key with the module name SchemaClasses.
+      # @return [Hash]
+      setting :schema_namespace_map, {}
     end
 
     # The configured metrics provider.

lib/deimos/utils/schema_class.rb

@@ -10,10 +10,25 @@ class << self
         # @return [Array<String>]
         def modules_for(namespace)
           modules = ['Schemas']
-          namespace_folder = namespace.split('.').last
-          if Deimos.config.schema.generate_namespace_folders && namespace_folder
-            modules.push(namespace_folder.underscore.classify)
+          namespace_override = nil
+          module_namespace = namespace
+
+          if Deimos.config.schema.use_full_namespace
+            if Deimos.config.schema.schema_namespace_map.present?
+              namespace_keys = Deimos.config.schema.schema_namespace_map.keys.sort_by { |k| -k.length }
+              namespace_override = namespace_keys.find { |k| module_namespace.include?(k) }
+            end
+
+            if namespace_override.present?
+              # override default module
+              modules = Array(Deimos.config.schema.schema_namespace_map[namespace_override])
+              module_namespace = module_namespace.gsub(/#{namespace_override}\.?/, '')
+            end
+
+            namespace_folders = module_namespace.split('.').map { |f| f.underscore.camelize }
+            modules.concat(namespace_folders) if namespace_folders.any?
           end
+
           modules
         end
 

lib/generators/deimos/schema_class_generator.rb

@@ -121,9 +121,16 @@ def write_file(schema, key_config)
           @main_class_definition = class_template
 
           file_prefix = schema.name.underscore.singularize
-          if Deimos.config.schema.generate_namespace_folders
-            file_prefix = "#{@modules.last.underscore.singularize}/#{file_prefix}"
+          if Deimos.config.schema.use_full_namespace
+            # Use entire namespace for folders
+            # but don't add directories that are already in the path
+            directories = @modules.map(&:underscore).select do |m|
+              Deimos.config.schema.generated_class_path.exclude?(m)
+            end
+
+            file_prefix = "#{directories.join('/')}/#{file_prefix}"
           end
+
           filename = "#{Deimos.config.schema.generated_class_path}/#{file_prefix}.rb"
           template(SCHEMA_CLASS_FILE, filename, force: true)
         end

regenerate_test_schema_classes.rb

@@ -21,8 +21,11 @@ def consume(payload, metadata); end
   deimos_config.schema.path = "spec/schemas"
   deimos_config.schema.backend = :avro_validation
   deimos_config.schema.generated_class_path = './spec/schemas'
-  deimos_config.schema.generate_namespace_folders = true
-  deimos_config.schema.nest_child_schemas = true
+  deimos_config.schema.use_full_namespace = true
+  deimos_config.schema.schema_namespace_map = {
+    'com' => 'Schemas',
+    'com.my-namespace.my-suborg' => %w(Schemas MyNamespace)
+  }
 
   consumer do
     class_name 'MyConsumer'
@@ -56,6 +59,14 @@ def consume(payload, metadata); end
     key_config field: :test_id
   end
 
+  consumer do
+    class_name 'MyConsumer'
+    topic 'MyTopic'
+    schema 'MyLongNamespaceSchema'
+    namespace 'com.my-namespace.my-suborg'
+    key_config field: :test_id
+  end
+
   producer do
     class_name 'MyConsumer'
     topic 'MyTopic'

spec/active_record_batch_consumer_spec.rb

@@ -78,7 +78,7 @@ def publish_batch(messages)
           before(:each) do
             Deimos.configure do |config|
               config.schema.use_schema_classes = use_schema_classes
-              config.schema.generate_namespace_folders = true
+              config.schema.use_full_namespace = true
             end
           end
 

spec/active_record_consumer_spec.rb

@@ -143,7 +143,7 @@ def as_json(_opts={})
           before(:each) do
             Deimos.configure do |config|
               config.schema.use_schema_classes = use_schema_classes
-              config.schema.generate_namespace_folders = true
+              config.schema.use_full_namespace = true
             end
           end
 

spec/active_record_producer_spec.rb

@@ -71,7 +71,7 @@ def self.post_process(batch)
         before(:each) do
           Deimos.configure do |config|
             config.schema.use_schema_classes = use_schema_classes
-            config.schema.generate_namespace_folders = true
+            config.schema.use_full_namespace = true
           end
         end
 

spec/batch_consumer_spec.rb

@@ -42,7 +42,7 @@ def consume_batch(_payloads, _metadata)
           before(:each) do
             Deimos.configure do |config|
               config.schema.use_schema_classes = use_schema_classes
-              config.schema.generate_namespace_folders = true
+              config.schema.use_full_namespace = true
             end
           end
 

spec/consumer_spec.rb

@@ -31,7 +31,7 @@ def consume(_payload, _metadata)
           before(:each) do
             Deimos.configure do |config|
               config.schema.use_schema_classes = use_schema_classes
-              config.schema.generate_namespace_folders = true
+              config.schema.use_full_namespace = true
             end
           end
 
@@ -138,7 +138,7 @@ def consume(_payload, _metadata)
         before(:each) do
           Deimos.configure do |config|
             config.schema.use_schema_classes = true
-            config.schema.generate_namespace_folders = true
+            config.schema.use_full_namespace = true
           end
         end
 

spec/generators/schema_class_generator_spec.rb

@@ -217,13 +217,30 @@ def dump(value)
 
     context 'with namespace folders' do
       it 'should generate the correct classes' do
-        Deimos.with_config('schema.generate_namespace_folders' => true) do
+        Deimos.with_config('schema.use_full_namespace' => true) do
           described_class.start
           expect(files).to match_snapshot('namespace_folders', snapshot_serializer: MultiFileSerializer)
         end
       end
     end
 
+    context 'with namespace map' do
+      it 'should generate the correct classes' do
+        Deimos.with_config(
+          {
+            'schema.use_full_namespace' => true,
+            'schema.schema_namespace_map' => {
+              'com' => 'Schemas',
+              'com.my-namespace.my-suborg' => %w(Schemas MyNamespace)
+            }
+          }
+        ) do
+          described_class.start
+          expect(files).to match_snapshot('namespace_map', snapshot_serializer: MultiFileSerializer)
+        end
+      end
+    end
+
     context 'nested true' do
       it 'should generate the correct classes' do
         Deimos.with_config('schema.nest_child_schemas' => true) do

spec/schemas/com/my-namespace/my-suborg/MyLongNamespaceSchema.avsc

@@ -0,0 +1,18 @@
+{
+  "namespace": "com.my-namespace.my-suborg",
+  "name": "MyLongNamespaceSchema",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "test_id",
+      "type": "string",
+      "doc": "test string"
+    },
+    {
+      "name": "some_int",
+      "type": "int",
+      "doc": "test int"
+    }
+  ]
+}

spec/schemas/my_namespace/generated.rb

@@ -128,7 +128,7 @@ def namespace
     end
 
     def self.tombstone(key)
-      record = self.new
+      record = self.allocate
       record.tombstone_key = key
       record.a_string = key
       record

spec/schemas/my_namespace/my_long_namespace_schema.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# This file is autogenerated by Deimos, Do NOT modify
+module Schemas; module MyNamespace
+  ### Primary Schema Class ###
+  # Autogenerated Schema for Record at com.my-namespace.my-suborg.MyLongNamespaceSchema
+  class MyLongNamespaceSchema < Deimos::SchemaClass::Record
+
+    ### Attribute Accessors ###
+    # @return [String]
+    attr_accessor :test_id
+    # @return [Integer]
+    attr_accessor :some_int
+
+    # @override
+    def initialize(test_id: nil,
+                   some_int: nil)
+      super
+      self.test_id = test_id
+      self.some_int = some_int
+    end
+
+    # @override
+    def schema
+      'MyLongNamespaceSchema'
+    end
+
+    # @override
+    def namespace
+      'com.my-namespace.my-suborg'
+    end
+
+    def self.tombstone(key)
+      record = self.allocate
+      record.tombstone_key = key
+      record.test_id = key
+      record
+    end
+
+    # @override
+    def as_json(_opts={})
+      {
+        'test_id' => @test_id,
+        'some_int' => @some_int
+      }
+    end
+  end
+end; end

spec/schemas/my_namespace/my_nested_schema.rb

@@ -104,7 +104,7 @@ def namespace
     end
 
     def self.tombstone(key)
-      record = self.new
+      record = self.allocate
       record.tombstone_key = key
       record.test_id = key
       record

spec/schemas/my_namespace/my_schema.rb

@@ -43,7 +43,7 @@ def namespace
     end
 
     def self.tombstone(key)
-      record = self.new
+      record = self.allocate
       record.tombstone_key = MySchemaKey.initialize_from_value(key)
       record.payload_key = key
       record

spec/schemas/my_namespace/my_schema_with_circular_reference.rb

@@ -68,7 +68,7 @@ def namespace
     end
 
     def self.tombstone(key)
-      record = self.new
+      record = self.allocate
       record
     end
 

spec/schemas/my_namespace/my_schema_with_complex_type.rb

@@ -9,7 +9,7 @@ class MySchemaWithComplexType < Deimos::SchemaClass::Record
     ### Secondary Schema Classes ###
     # Autogenerated Schema for Record at com.my-namespace.ARecord
     class ARecord < Deimos::SchemaClass::Record
-
+  
       ### Attribute Accessors ###
       # @return [String]
       attr_accessor :a_record_field
@@ -185,7 +185,7 @@ def namespace
     end
 
     def self.tombstone(key)
-      record = self.new
+      record = self.allocate
       record.tombstone_key = key
       record.test_id = key
       record

spec/schemas/my_namespace/my_updated_schema.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # This file is autogenerated by Deimos, Do NOT modify
-module Schemas; module MyNamespace
+module Schemas; module Com; module MyNamespace
   ### Primary Schema Class ###
   # Autogenerated Schema for Record at com.my-namespace.MySchema
   class MyUpdatedSchema < Schemas::MyNamespace::MySchema
@@ -16,3 +16,4 @@ def initialize(test_id: nil,
   end
 end
 end
+end

spec/schemas/request/create_topic.rb → spec/schemas/my_namespace/request/create_topic.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # This file is autogenerated by Deimos, Do NOT modify
-module Schemas; module Request
+module Schemas; module MyNamespace; module Request
   ### Primary Schema Class ###
   # Autogenerated Schema for Record at com.my-namespace.request.CreateTopic
   class CreateTopic < Deimos::SchemaClass::Record
@@ -33,4 +33,4 @@ def as_json(_opts={})
       }
     end
   end
-end; end
+end; end; end

spec/schemas/request/index.rb → spec/schemas/my_namespace/request/index.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # This file is autogenerated by Deimos, Do NOT modify
-module Schemas; module Request
+module Schemas; module MyNamespace; module Request
   ### Primary Schema Class ###
   # Autogenerated Schema for Record at com.my-namespace.request.Index
   class Index < Deimos::SchemaClass::Record
@@ -33,4 +33,4 @@ def as_json(_opts={})
       }
     end
   end
-end; end
+end; end; end

spec/schemas/request/update_request.rb → spec/schemas/my_namespace/request/update_request.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # This file is autogenerated by Deimos, Do NOT modify
-module Schemas; module Request
+module Schemas; module MyNamespace; module Request
   ### Primary Schema Class ###
   # Autogenerated Schema for Record at com.my-namespace.request.UpdateRequest
   class UpdateRequest < Deimos::SchemaClass::Record
@@ -33,4 +33,4 @@ def as_json(_opts={})
       }
     end
   end
-end; end
+end; end; end

spec/schemas/response/create_topic.rb → spec/schemas/my_namespace/response/create_topic.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # This file is autogenerated by Deimos, Do NOT modify
-module Schemas; module Response
+module Schemas; module MyNamespace; module Response
   ### Primary Schema Class ###
   # Autogenerated Schema for Record at com.my-namespace.response.CreateTopic
   class CreateTopic < Deimos::SchemaClass::Record
@@ -33,4 +33,4 @@ def as_json(_opts={})
       }
     end
   end
-end; end
+end; end; end