From 13f1f6a2f87feb903162455c27ffb3aa18e46f6e Mon Sep 17 00:00:00 2001 From: Adam Gulczynski Date: Wed, 26 Jul 2023 19:31:43 +0200 Subject: [PATCH 1/4] setup elasticsearch - docker, connection (#70) --- README.md | 16 ++++++++-- app.env.sample | 1 + db/queries/job.sql | 13 ++++++++ db/queries/job_skill.sql | 5 +++ db/sqlc/job.sql.go | 59 ++++++++++++++++++++++++++++++++++ db/sqlc/job_skill.sql.go | 29 +++++++++++++++++ db/sqlc/querier.go | 2 ++ docker-compose.yml | 46 +++++++++++++++++++++++++++ esearch/connect.go | 21 ++++++++++++ esearch/index.go | 62 ++++++++++++++++++++++++++++++++++++ esearch/load.go | 69 ++++++++++++++++++++++++++++++++++++++++ esearch/types.go | 21 ++++++++++++ go.mod | 9 +++--- go.sum | 29 +++-------------- main.go | 7 ++++ utils/config.go | 11 ++++--- 16 files changed, 363 insertions(+), 37 deletions(-) create mode 100644 esearch/connect.go create mode 100644 esearch/index.go create mode 100644 esearch/load.go create mode 100644 esearch/types.go diff --git a/README.md b/README.md index 40920e0..6b92e90 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ - [Viper](https://github.com/spf13/viper) - [gin-swagger](https://github.com/swaggo/gin-swagger) for generating docs +
+ ## Getting started 1. Clone the repository 2. Go to the project's root directory @@ -22,7 +24,7 @@ - `docker-compose up` to run the database container - `make runserver` - to run HTTP server 5. Now everything should be ready and server running on `SERVER_ADDRESS` specified in `app.env` - +
## Testing 1. Run the postgres container (`docker-compose up`) @@ -32,7 +34,7 @@ - `make test_coverage p={PATH}` - to get the coverage in the HTML format - where `{PATH}` is the path to the target directory for which you want to generate test coverage. The `{PATH}` should be replaced with the actual path you want to use. For example `./api` or - use standard `go test` commands (e.g. `go test -v ./api`) - +
## API endpoints This API provides a set of endpoints for managing: @@ -40,7 +42,15 @@ This API provides a set of endpoints for managing: - employers - jobs -#### The base path for all endpoints is `/api/v1`. +After running the server, the Swagger documentation is available at http://localhost:8080/swagger/index.html. +You can find there detailed information about the API endpoints, including their parameters, +request and response formats, and examples. You can use the Swagger UI to test the API +endpoints and see their responses in real-time. + +### The base path for all endpoints is `/api/v1` +so for example `/api/v1/users/login` + + Here is a summary of the available endpoints and their functionality: diff --git a/app.env.sample b/app.env.sample index 3879682..7b3321c 100644 --- a/app.env.sample +++ b/app.env.sample @@ -1,5 +1,6 @@ DB_DRIVER=postgres DB_SOURCE=based on docker-compose.yml -> postgresql://devuser:admin@localhost:5432/go_gin_job_search_db?sslmode=disable SERVER_ADDRESS=for example 0.0.0.0:8080 +ELASTICSEARCH_ADDRESS=for example 0.0.0.0:9200 TOKEN_SYMMETRIC_KEY=32 characters long, you can use just 12345678901234567890123456789012 ACCESS_TOKEN_DURATION=for example 20m or 24h \ No newline at end of file diff --git a/db/queries/job.sql b/db/queries/job.sql index 722b4ac..1454f7e 100644 --- a/db/queries/job.sql +++ b/db/queries/job.sql @@ -106,3 +106,16 @@ RETURNING *; DELETE FROM jobs WHERE id = $1; + +-- name: ListAllJobsForES :many +SELECT j.id, + j.title, + j.industry, + j.location, + j.description, + c.name AS company_name, + j.salary_min, + j.salary_max, + j.requirements +FROM jobs j + JOIN companies c ON j.company_id = c.id; diff --git a/db/queries/job_skill.sql b/db/queries/job_skill.sql index 6bdf551..25e45a7 100644 --- a/db/queries/job_skill.sql +++ b/db/queries/job_skill.sql @@ -9,6 +9,11 @@ FROM job_skills WHERE job_id = $1 LIMIT $2 OFFSET $3; +-- name: ListAllJobSkillsByJobID :many +SELECT skill +FROM job_skills +WHERE job_id = $1; + -- name: ListJobsBySkill :many SELECT job_id FROM job_skills diff --git a/db/sqlc/job.sql.go b/db/sqlc/job.sql.go index 3f07a45..28e64ef 100644 --- a/db/sqlc/job.sql.go +++ b/db/sqlc/job.sql.go @@ -153,6 +153,65 @@ func (q *Queries) GetJobDetails(ctx context.Context, id int32) (GetJobDetailsRow return i, err } +const listAllJobsForES = `-- name: ListAllJobsForES :many +SELECT j.id, + j.title, + j.industry, + j.location, + j.description, + c.name AS company_name, + j.salary_min, + j.salary_max, + j.requirements +FROM jobs j + JOIN companies c ON j.company_id = c.id +` + +type ListAllJobsForESRow struct { + ID int32 `json:"id"` + Title string `json:"title"` + Industry string `json:"industry"` + Location string `json:"location"` + Description string `json:"description"` + CompanyName string `json:"company_name"` + SalaryMin int32 `json:"salary_min"` + SalaryMax int32 `json:"salary_max"` + Requirements string `json:"requirements"` +} + +func (q *Queries) ListAllJobsForES(ctx context.Context) ([]ListAllJobsForESRow, error) { + rows, err := q.db.QueryContext(ctx, listAllJobsForES) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListAllJobsForESRow{} + for rows.Next() { + var i ListAllJobsForESRow + if err := rows.Scan( + &i.ID, + &i.Title, + &i.Industry, + &i.Location, + &i.Description, + &i.CompanyName, + &i.SalaryMin, + &i.SalaryMax, + &i.Requirements, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listJobsByCompanyExactName = `-- name: ListJobsByCompanyExactName :many SELECT j.id, j.title, j.industry, j.company_id, j.description, j.location, j.salary_min, j.salary_max, j.requirements, j.created_at, c.name AS company_name diff --git a/db/sqlc/job_skill.sql.go b/db/sqlc/job_skill.sql.go index 1164129..f364810 100644 --- a/db/sqlc/job_skill.sql.go +++ b/db/sqlc/job_skill.sql.go @@ -62,6 +62,35 @@ func (q *Queries) DeleteMultipleJobSkills(ctx context.Context, ids []int32) erro return err } +const listAllJobSkillsByJobID = `-- name: ListAllJobSkillsByJobID :many +SELECT skill +FROM job_skills +WHERE job_id = $1 +` + +func (q *Queries) ListAllJobSkillsByJobID(ctx context.Context, jobID int32) ([]string, error) { + rows, err := q.db.QueryContext(ctx, listAllJobSkillsByJobID, jobID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []string{} + for rows.Next() { + var skill string + if err := rows.Scan(&skill); err != nil { + return nil, err + } + items = append(items, skill) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listJobSkillsByJobID = `-- name: ListJobSkillsByJobID :many SELECT id, skill FROM job_skills diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 4f9595b..a7d9d4b 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -33,6 +33,8 @@ type Querier interface { GetJobDetails(ctx context.Context, id int32) (GetJobDetailsRow, error) GetUserByEmail(ctx context.Context, email string) (User, error) GetUserByID(ctx context.Context, id int32) (User, error) + ListAllJobSkillsByJobID(ctx context.Context, jobID int32) ([]string, error) + ListAllJobsForES(ctx context.Context) ([]ListAllJobsForESRow, error) ListJobSkillsByJobID(ctx context.Context, arg ListJobSkillsByJobIDParams) ([]ListJobSkillsByJobIDRow, error) ListJobsByCompanyExactName(ctx context.Context, arg ListJobsByCompanyExactNameParams) ([]ListJobsByCompanyExactNameRow, error) ListJobsByCompanyID(ctx context.Context, arg ListJobsByCompanyIDParams) ([]ListJobsByCompanyIDRow, error) diff --git a/docker-compose.yml b/docker-compose.yml index 9514eec..d97fb12 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,5 +13,51 @@ services: ports: - "5432:5432" + elasticsearch: + container_name: go_gin_job_search_es + image: elasticsearch:7.17.9 + environment: + - bootstrap.memory_lock=true + - ES_JAVA_OPTS=-Xms1g -Xmx1g + - cluster.name=job-search-esearch + - discovery.type=single-node + - node.name=job-search-es-node + ulimits: + memlock: + hard: -1 + soft: -1 + ports: + - "9200:9200" + networks: + - es-job-search + healthcheck: + interval: 10s + retries: 20 + test: curl -s http://localhost:9200/_cluster/health | grep -vq '"status":"red"' + depends_on: + - db + + kibana: + image: kibana:7.17.10 + container_name: go_gin_job_search_kibana + depends_on: + elasticsearch: + condition: service_healthy + environment: + ELASTICSEARCH_URL: http://elasticsearch:9200 + ELASTICSEARCH_HOSTS: http://elasticsearch:9200 + ports: + - "5601:5601" + networks: + - es-job-search + healthcheck: + interval: 10s + retries: 20 + test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:5601/api/status + +networks: + es-job-search: + driver: bridge + volumes: dev-db-data: \ No newline at end of file diff --git a/esearch/connect.go b/esearch/connect.go new file mode 100644 index 0000000..4b8cef9 --- /dev/null +++ b/esearch/connect.go @@ -0,0 +1,21 @@ +package esearch + +import ( + "context" + "github.com/elastic/go-elasticsearch/v8" +) + +// ConnectWithElasticsearch creates a new elasticsearch client and stores it in the context +func ConnectWithElasticsearch(ctx context.Context, address string) context.Context { + + newClient, err := elasticsearch.NewClient(elasticsearch.Config{ + Addresses: []string{ + address, + }, + }) + if err != nil { + panic(err) + } + + return context.WithValue(ctx, ClientKey, newClient) +} diff --git a/esearch/index.go b/esearch/index.go new file mode 100644 index 0000000..1d62781 --- /dev/null +++ b/esearch/index.go @@ -0,0 +1,62 @@ +package esearch + +import ( + "bytes" + "context" + "io" + "log" + "strconv" + + "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/esutil" +) + +// IndexJobsAsDocuments index jobs as documents +func IndexJobsAsDocuments(ctx context.Context) { + + jobs := ctx.Value(JobKey).([]Job) + client := ctx.Value(ClientKey).(*elasticsearch.Client) + + bulkIndexer, err := esutil.NewBulkIndexer(esutil.BulkIndexerConfig{ + Index: "jobs", + Client: client, + NumWorkers: 5, + }) + if err != nil { + panic(err) + } + + for documentID, document := range jobs { + body, err := convertToReadSeeker(esutil.NewJSONReader(document)) + if err != nil { + panic(err) + } + err = bulkIndexer.Add( + ctx, + esutil.BulkIndexerItem{ + Action: "index", + DocumentID: strconv.Itoa(documentID), + Body: body, + }, + ) + if err != nil { + panic(err) + } + } + + bulkIndexer.Close(ctx) + biStats := bulkIndexer.Stats() + log.Printf("Jobs indexed on Elasticsearch: %d \n", biStats.NumIndexed) +} + +func convertToReadSeeker(reader io.Reader) (io.ReadSeeker, error) { + // Read the entire content of the reader into a buffer. + data, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + + // Create a new io.ReadSeeker from the buffer. + readSeeker := bytes.NewReader(data) + return readSeeker, nil +} diff --git a/esearch/load.go b/esearch/load.go new file mode 100644 index 0000000..678e7bc --- /dev/null +++ b/esearch/load.go @@ -0,0 +1,69 @@ +package esearch + +import ( + "context" + db "github.com/aalug/go-gin-job-search/db/sqlc" + "log" + "sync" +) + +func LoadJobsFromDB(ctx context.Context, store db.Store) context.Context { + const ( + concurrency = 5 + ) + + var ( + jobs []Job + waitGroup = new(sync.WaitGroup) + workQueue = make(chan Job) + mutex = &sync.Mutex{} + ) + + // Fetch jobs from the database + jobsFromDB, err := store.ListAllJobsForES(ctx) + if err != nil { + panic(err) + } + + // Populate the work queue with movies from the database. + go func() { + for _, job := range jobsFromDB { + skills, err := store.ListAllJobSkillsByJobID(ctx, job.ID) + if err != nil { + panic(err) + } + j := Job{ + ID: job.ID, + Title: job.Title, + Industry: job.Industry, + CompanyName: job.CompanyName, + Description: job.Description, + Location: job.Location, + SalaryMin: job.SalaryMin, + SalaryMax: job.SalaryMax, + Requirements: job.Requirements, + JobSkills: skills, + } + workQueue <- j + } + close(workQueue) + }() + + for i := 0; i < concurrency; i++ { + waitGroup.Add(1) + go func(workQueue chan Job, waitGroup *sync.WaitGroup) { + for job := range workQueue { + mutex.Lock() + jobs = append(jobs, job) + mutex.Unlock() + } + waitGroup.Done() + }(workQueue, waitGroup) + } + + waitGroup.Wait() + + log.Printf("Jobs loaded from the database: %d\n", len(jobs)) + return context.WithValue(ctx, JobKey, jobs) + +} diff --git a/esearch/types.go b/esearch/types.go new file mode 100644 index 0000000..511817b --- /dev/null +++ b/esearch/types.go @@ -0,0 +1,21 @@ +package esearch + +type Job struct { + ID int32 `json:"id"` + Title string `json:"title"` + Industry string `json:"industry"` + CompanyName string `json:"company_name"` + Description string `json:"description"` + Location string `json:"location"` + SalaryMin int32 `json:"salary_min"` + SalaryMax int32 `json:"salary_max"` + Requirements string `json:"requirements"` + JobSkills []string `json:"job_skills"` +} + +type contextKey struct { + Key int +} + +var JobKey contextKey = contextKey{Key: 1} +var ClientKey contextKey = contextKey{Key: 2} diff --git a/go.mod b/go.mod index cc2494d..7ccf2c8 100644 --- a/go.mod +++ b/go.mod @@ -12,14 +12,14 @@ require ( github.com/o1egl/paseto v1.0.0 github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 + github.com/swaggo/files v1.0.1 + github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.1 golang.org/x/crypto v0.11.0 ) require ( github.com/KyleBanks/depth v1.2.1 // indirect - github.com/PuerkitoBio/purell v1.2.0 // indirect - github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect github.com/aead/chacha20poly1305 v0.0.0-20170617001512-233f39982aeb // indirect github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 // indirect @@ -27,6 +27,8 @@ require ( github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/elastic/elastic-transport-go/v8 v8.3.0 // indirect + github.com/elastic/go-elasticsearch/v8 v8.8.2 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect @@ -57,8 +59,6 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.4.2 // indirect - github.com/swaggo/files v1.0.1 // indirect - github.com/swaggo/gin-swagger v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect golang.org/x/arch v0.4.0 // indirect @@ -68,6 +68,5 @@ require ( golang.org/x/tools v0.11.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 89c26f5..2165869 100644 --- a/go.sum +++ b/go.sum @@ -40,12 +40,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.2.0 h1:/Jdm5QfyM8zdlqT6WVZU4cfP23sot6CEHA4CS49Ezig= -github.com/PuerkitoBio/purell v1.2.0/go.mod h1:OhLRTaaIzhvIyofkJfB24gokC7tM42Px5UhoT32THBk= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY= github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA= github.com/aead/chacha20poly1305 v0.0.0-20170617001512-233f39982aeb h1:6Z/wqhPFZ7y5ksCEV/V5MXOazLaeu/EW97CU5rz8NWk= @@ -53,14 +47,11 @@ github.com/aead/chacha20poly1305 v0.0.0-20170617001512-233f39982aeb/go.mod h1:Uz github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 h1:52m0LGchQBBVqJRyYYufQuIbVqRawmubW3OFGqK1ekw= github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635/go.mod h1:lmLxL+FV291OopO93Bwf9fQLQeLyt33VJRUg5VJ30us= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.9.2 h1:GDaNjuWSGu09guE9Oql0MSTNhNCLlWwO8y/xM5BzcbM= -github.com/bytedance/sonic v1.9.2/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= github.com/bytedance/sonic v1.10.0-rc2 h1:oDfRZ+4m6AYCOC0GFeOCeYqvBmucy1isvouS2K0cPzo= github.com/bytedance/sonic v1.10.0-rc2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0= github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= @@ -77,6 +68,10 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elastic/elastic-transport-go/v8 v8.3.0 h1:DJGxovyQLXGr62e9nDMPSxRyWION0Bh6d9eCFBriiHo= +github.com/elastic/elastic-transport-go/v8 v8.3.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= +github.com/elastic/go-elasticsearch/v8 v8.8.2 h1:3ITzPlRNadzDnbLTnMRjrAN4j4G3LvFo5gCIWDPS6pY= +github.com/elastic/go-elasticsearch/v8 v8.8.2/go.mod h1:GU1BJHO7WeamP7UhuElYwzzHtvf9SDmeVpSSy9+o6Qg= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -90,6 +85,7 @@ github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g= github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs= +github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= @@ -99,22 +95,16 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= -github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= -github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= -github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= github.com/go-openapi/spec v0.20.9 h1:xnlYNQAwKd2VQRRfwTEI0DcK+2cbuvI/0c7jx3gA8/8= github.com/go-openapi/spec v0.20.9/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= @@ -229,7 +219,6 @@ github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0V github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= @@ -247,8 +236,6 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA github.com/o1egl/paseto v1.0.0 h1:bwpvPu2au176w4IBlhbyUv/S5VPptERIA99Oap5qUd0= github.com/o1egl/paseto v1.0.0/go.mod h1:5HxsZPmw/3RI2pAwGo1HhOOwSdvBpcuVzO7uDkm+CLU= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= -github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= -github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -286,7 +273,6 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= @@ -365,7 +351,6 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -399,7 +384,6 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -460,7 +444,6 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -540,8 +523,6 @@ golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go index c1a243a..0706df6 100644 --- a/main.go +++ b/main.go @@ -1,9 +1,11 @@ package main import ( + "context" "database/sql" "github.com/aalug/go-gin-job-search/api" db "github.com/aalug/go-gin-job-search/db/sqlc" + "github.com/aalug/go-gin-job-search/esearch" "github.com/aalug/go-gin-job-search/utils" "log" ) @@ -21,6 +23,11 @@ func main() { store := db.NewStore(conn) + // Elasticsearch + ctx := context.Background() + ctx = esearch.LoadJobsFromDB(ctx, store) + ctx = esearch.ConnectWithElasticsearch(ctx, config.ElasticSearchAddress) + // @BasePath /api/v1 // @contact.name aalug // @contact.url https://github.com/aalug diff --git a/utils/config.go b/utils/config.go index f9284b1..17c016f 100644 --- a/utils/config.go +++ b/utils/config.go @@ -7,11 +7,12 @@ import ( // Config stores configuration of the application type Config struct { - DBDriver string `mapstructure:"DB_DRIVER"` - DBSource string `mapstructure:"DB_SOURCE"` - ServerAddress string `mapstructure:"SERVER_ADDRESS"` - TokenSymmetricKey string `mapstructure:"TOKEN_SYMMETRIC_KEY"` - AccessTokenDuration time.Duration `mapstructure:"ACCESS_TOKEN_DURATION"` + DBDriver string `mapstructure:"DB_DRIVER"` + DBSource string `mapstructure:"DB_SOURCE"` + ServerAddress string `mapstructure:"SERVER_ADDRESS"` + ElasticSearchAddress string `mapstructure:"ELASTICSEARCH_ADDRESS"` + TokenSymmetricKey string `mapstructure:"TOKEN_SYMMETRIC_KEY"` + AccessTokenDuration time.Duration `mapstructure:"ACCESS_TOKEN_DURATION"` } func LoadConfig(path string) (config Config, err error) { From 638418e0d362781649b564521df1f0675d5d6007 Mon Sep 17 00:00:00 2001 From: Adam Gulczynski Date: Thu, 27 Jul 2023 00:09:32 +0200 Subject: [PATCH 2/4] create function to query jobs index by document ID (#71) --- app.env.sample | 2 +- esearch/index.go | 17 +++++++++++++++-- esearch/lookup.go | 29 +++++++++++++++++++++++++++++ esearch/types.go | 12 ++++++++++++ main.go | 10 ++++++++-- 5 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 esearch/lookup.go diff --git a/app.env.sample b/app.env.sample index 7b3321c..41b505b 100644 --- a/app.env.sample +++ b/app.env.sample @@ -1,6 +1,6 @@ DB_DRIVER=postgres DB_SOURCE=based on docker-compose.yml -> postgresql://devuser:admin@localhost:5432/go_gin_job_search_db?sslmode=disable SERVER_ADDRESS=for example 0.0.0.0:8080 -ELASTICSEARCH_ADDRESS=for example 0.0.0.0:9200 +ELASTICSEARCH_ADDRESS=for example http://localhost:9200 TOKEN_SYMMETRIC_KEY=32 characters long, you can use just 12345678901234567890123456789012 ACCESS_TOKEN_DURATION=for example 20m or 24h \ No newline at end of file diff --git a/esearch/index.go b/esearch/index.go index 1d62781..e531a40 100644 --- a/esearch/index.go +++ b/esearch/index.go @@ -27,7 +27,7 @@ func IndexJobsAsDocuments(ctx context.Context) { } for documentID, document := range jobs { - body, err := convertToReadSeeker(esutil.NewJSONReader(document)) + body, err := readerToReadSeeker(esutil.NewJSONReader(document)) if err != nil { panic(err) } @@ -49,7 +49,20 @@ func IndexJobsAsDocuments(ctx context.Context) { log.Printf("Jobs indexed on Elasticsearch: %d \n", biStats.NumIndexed) } -func convertToReadSeeker(reader io.Reader) (io.ReadSeeker, error) { +// IndexJobAsDocument index one job as document +//func IndexJobAsDocument(ctx context.Context, job Job) { + +//client := ctx.Value(ClientKey).(*elasticsearch.Client) + +// get id of the last document and set documentID to it + 1 +//_, err := client.Index("movies", esutil.NewJSONReader(job), +// client.Index.WithDocumentID(strconv.Itoa(documentID))) +//if err != nil { +// panic(err) +//} +//} + +func readerToReadSeeker(reader io.Reader) (io.ReadSeeker, error) { // Read the entire content of the reader into a buffer. data, err := io.ReadAll(reader) if err != nil { diff --git a/esearch/lookup.go b/esearch/lookup.go new file mode 100644 index 0000000..d6ee69f --- /dev/null +++ b/esearch/lookup.go @@ -0,0 +1,29 @@ +package esearch + +import ( + "context" + "encoding/json" + "github.com/elastic/go-elasticsearch/v8" + "strconv" +) + +// QueryJobsByDocumentID queries the jobs index by document ID. +// This is a helper function for testing purposes. +func QueryJobsByDocumentID(ctx context.Context, documentID int) *Job { + + client := ctx.Value(ClientKey).(*elasticsearch.Client) + + response, err := client.Get("jobs", strconv.Itoa(documentID)) + if err != nil { + panic(err) + } + defer response.Body.Close() + + var getResponse = GetResponse{} + err = json.NewDecoder(response.Body).Decode(&getResponse) + if err != nil { + panic(err) + } + + return getResponse.Source +} diff --git a/esearch/types.go b/esearch/types.go index 511817b..5ba6827 100644 --- a/esearch/types.go +++ b/esearch/types.go @@ -1,5 +1,7 @@ package esearch +// === Types for the ES part of the Application === + type Job struct { ID int32 `json:"id"` Title string `json:"title"` @@ -13,9 +15,19 @@ type Job struct { JobSkills []string `json:"job_skills"` } +// === for the Context === type contextKey struct { Key int } var JobKey contextKey = contextKey{Key: 1} var ClientKey contextKey = contextKey{Key: 2} + +// === Queries and Searches === + +type GetResponse struct { + Index string `json:"_index"` + ID string `json:"_id"` + Version int `json:"_version"` + Source *Job `json:"_source"` +} diff --git a/main.go b/main.go index 0706df6..6aaeecc 100644 --- a/main.go +++ b/main.go @@ -11,11 +11,13 @@ import ( ) func main() { + // === config, env file === config, err := utils.LoadConfig(".") if err != nil { log.Fatal("cannot load env file: ", err) } + // === database === conn, err := sql.Open(config.DBDriver, config.DBSource) if err != nil { log.Fatal("cannot connect to the db: ", err) @@ -23,11 +25,15 @@ func main() { store := db.NewStore(conn) - // Elasticsearch + // === Elasticsearch === ctx := context.Background() - ctx = esearch.LoadJobsFromDB(ctx, store) + // TODO: for now, all jobs are indexed every time the server starts + // TODO: later on, we will index only new or updated jobs + //ctx = esearch.LoadJobsFromDB(ctx, store) ctx = esearch.ConnectWithElasticsearch(ctx, config.ElasticSearchAddress) + esearch.IndexJobsAsDocuments(ctx) + // === HTTP server === // @BasePath /api/v1 // @contact.name aalug // @contact.url https://github.com/aalug From 582850a39a0d61f7a428f377c2a2dd9f65fbfb06 Mon Sep 17 00:00:00 2001 From: Adam Gulczynski Date: Thu, 27 Jul 2023 17:08:02 +0200 Subject: [PATCH 3/4] handle search jobs using Elasticsearch (#72) --- esearch/lookup.go | 3 +- esearch/search.go | 73 +++++++++++++++++++++++++++++++++++++++++++++++ esearch/types.go | 15 ++++++++-- main.go | 8 +++++- 4 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 esearch/search.go diff --git a/esearch/lookup.go b/esearch/lookup.go index d6ee69f..421c39e 100644 --- a/esearch/lookup.go +++ b/esearch/lookup.go @@ -3,8 +3,9 @@ package esearch import ( "context" "encoding/json" - "github.com/elastic/go-elasticsearch/v8" "strconv" + + "github.com/elastic/go-elasticsearch/v8" ) // QueryJobsByDocumentID queries the jobs index by document ID. diff --git a/esearch/search.go b/esearch/search.go new file mode 100644 index 0000000..aa0062e --- /dev/null +++ b/esearch/search.go @@ -0,0 +1,73 @@ +package esearch + +import ( + "bytes" + "context" + "encoding/json" + "github.com/elastic/go-elasticsearch/v8" +) + +func SearchJobs(ctx context.Context, query string) []*Job { + + client := ctx.Value(ClientKey).(*elasticsearch.Client) + + var searchBuffer bytes.Buffer + search := map[string]interface{}{ + "query": map[string]interface{}{ + "bool": map[string]interface{}{ + "should": []interface{}{ + map[string]interface{}{ + "match": map[string]interface{}{ + "title": map[string]interface{}{ + "query": query, + "fuzziness": "AUTO", + }, + }, + }, + map[string]interface{}{ + "multi_match": map[string]interface{}{ + "query": query, + "fields": []string{ + "description", + "requirements", + "job_skills", + "location", + }, + "fuzziness": "AUTO", + }, + }, + }, + }, + }, + } + err := json.NewEncoder(&searchBuffer).Encode(search) + if err != nil { + panic(err) + } + + response, err := client.Search( + client.Search.WithContext(ctx), + client.Search.WithIndex("jobs"), + client.Search.WithBody(&searchBuffer), + client.Search.WithTrackTotalHits(true), + client.Search.WithPretty(), + ) + if err != nil { + panic(err) + } + defer response.Body.Close() + + var searchResponse = SearchResponse{} + err = json.NewDecoder(response.Body).Decode(&searchResponse) + if err != nil { + panic(err) + } + + var jobs []*Job + if searchResponse.Hits.Total.Value > 0 { + for _, job := range searchResponse.Hits.Hits { + jobs = append(jobs, job.Source) + } + } + return jobs +} diff --git a/esearch/types.go b/esearch/types.go index 5ba6827..0f6eb51 100644 --- a/esearch/types.go +++ b/esearch/types.go @@ -20,8 +20,8 @@ type contextKey struct { Key int } -var JobKey contextKey = contextKey{Key: 1} -var ClientKey contextKey = contextKey{Key: 2} +var JobKey = contextKey{Key: 1} +var ClientKey = contextKey{Key: 2} // === Queries and Searches === @@ -31,3 +31,14 @@ type GetResponse struct { Version int `json:"_version"` Source *Job `json:"_source"` } + +type SearchResponse struct { + Hits struct { + Total struct { + Value int64 `json:"value"` + } `json:"total"` + Hits []*struct { + Source *Job `json:"_source"` + } `json:"hits"` + } `json:"hits"` +} diff --git a/main.go b/main.go index 6aaeecc..0caf2b4 100644 --- a/main.go +++ b/main.go @@ -31,7 +31,13 @@ func main() { // TODO: later on, we will index only new or updated jobs //ctx = esearch.LoadJobsFromDB(ctx, store) ctx = esearch.ConnectWithElasticsearch(ctx, config.ElasticSearchAddress) - esearch.IndexJobsAsDocuments(ctx) + //esearch.IndexJobsAsDocuments(ctx) + jobs := esearch.SearchJobs(ctx, "descriptor") + log.Println("*****************") + for _, job := range jobs { + log.Println(job.Title) + } + log.Println("*****************") // === HTTP server === // @BasePath /api/v1 From 8e5770504c773965246194bc060324f3640b6e0f Mon Sep 17 00:00:00 2001 From: Adam Gulczynski Date: Thu, 27 Jul 2023 17:19:59 +0200 Subject: [PATCH 4/4] handle search jobs using Elasticsearch (#72) --- db/mock/store.go | 30 ++++++++++++++++++++++++++++++ main.go | 7 +------ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/db/mock/store.go b/db/mock/store.go index 96c8577..0d6a46f 100644 --- a/db/mock/store.go +++ b/db/mock/store.go @@ -444,6 +444,36 @@ func (mr *MockStoreMockRecorder) GetUserDetailsByEmail(arg0, arg1 interface{}) * return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUserDetailsByEmail", reflect.TypeOf((*MockStore)(nil).GetUserDetailsByEmail), arg0, arg1) } +// ListAllJobSkillsByJobID mocks base method. +func (m *MockStore) ListAllJobSkillsByJobID(arg0 context.Context, arg1 int32) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListAllJobSkillsByJobID", arg0, arg1) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListAllJobSkillsByJobID indicates an expected call of ListAllJobSkillsByJobID. +func (mr *MockStoreMockRecorder) ListAllJobSkillsByJobID(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListAllJobSkillsByJobID", reflect.TypeOf((*MockStore)(nil).ListAllJobSkillsByJobID), arg0, arg1) +} + +// ListAllJobsForES mocks base method. +func (m *MockStore) ListAllJobsForES(arg0 context.Context) ([]db.ListAllJobsForESRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListAllJobsForES", arg0) + ret0, _ := ret[0].([]db.ListAllJobsForESRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListAllJobsForES indicates an expected call of ListAllJobsForES. +func (mr *MockStoreMockRecorder) ListAllJobsForES(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListAllJobsForES", reflect.TypeOf((*MockStore)(nil).ListAllJobsForES), arg0) +} + // ListJobSkillsByJobID mocks base method. func (m *MockStore) ListJobSkillsByJobID(arg0 context.Context, arg1 db.ListJobSkillsByJobIDParams) ([]db.ListJobSkillsByJobIDRow, error) { m.ctrl.T.Helper() diff --git a/main.go b/main.go index 0caf2b4..f6eb549 100644 --- a/main.go +++ b/main.go @@ -32,12 +32,7 @@ func main() { //ctx = esearch.LoadJobsFromDB(ctx, store) ctx = esearch.ConnectWithElasticsearch(ctx, config.ElasticSearchAddress) //esearch.IndexJobsAsDocuments(ctx) - jobs := esearch.SearchJobs(ctx, "descriptor") - log.Println("*****************") - for _, job := range jobs { - log.Println(job.Title) - } - log.Println("*****************") + //jobs := esearch.SearchJobs(ctx, "descriptor") // === HTTP server === // @BasePath /api/v1