-
Notifications
You must be signed in to change notification settings - Fork 9
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
OplogToRedis: Persistent Denylist (#79)
Persist the configurable denylist across a restart. The service takes an optional environment variable configuration option `OTR_PG_PERSISTENCE_URL` to a postgres database. The service will create a table `otr_denylist` with one column (a list of unique denylist entries). The service will write to the table whenever entries are created or removed. Since all OTR replicas should have the same denylist, this should be fine- first to add adds, and first to delete deletes. The service will read the table contents at startup to populate the initial denylist. This should make the service resilient to restarting after a denylist entry was added. The HTTP methods should only respond 2xx after writing to the DB, so persistence failures will be known by the calling context.
- Loading branch information
1 parent
0fc3b98
commit f7a2b89
Showing
13 changed files
with
310 additions
and
63 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
74 changes: 74 additions & 0 deletions
74
integration-tests/fault-injection/denylist_persistence_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "reflect" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/tulip/oplogtoredis/integration-tests/fault-injection/harness" | ||
| "github.com/tulip/oplogtoredis/integration-tests/helpers" | ||
| ) | ||
|
|
||
| // This test restarts oplogtoredis after adding denylist entries. | ||
| // We expect the entries to still be there afterward, and be removable. | ||
| func TestDenylistPersistence(t *testing.T) { | ||
| mongo := harness.StartMongoServer() | ||
| defer mongo.Stop() | ||
|
|
||
| // Sleeping here for a while as the initial connection seems to be unreliable | ||
| time.Sleep(time.Second * 1) | ||
|
|
||
| redis := harness.StartRedisServer() | ||
| defer redis.Stop() | ||
|
|
||
| pg := harness.StartPostgresServer() | ||
| defer pg.Stop() | ||
|
|
||
| // wait before starting OTR for the auth changes to take effects | ||
| time.Sleep(3 * time.Second) | ||
|
|
||
| otr := harness.StartOTRProcessWithEnv(mongo.Addr, redis.Addr, 9000, []string{ | ||
| fmt.Sprintf("OTR_PG_PERSISTENCE_URL=%s", pg.ConnStr), | ||
| }) | ||
| defer otr.Stop() | ||
|
|
||
| time.Sleep(3 * time.Second) | ||
|
|
||
| baseURL := "http://localhost:9000" | ||
| // PUT new rule | ||
| helpers.DoRequest("PUT", baseURL, "/denylist/abc", t, 201) | ||
| // PUT second rule | ||
| helpers.DoRequest("PUT", baseURL, "/denylist/def", t, 201) | ||
| // GET list with both rules | ||
| data := helpers.DoRequest("GET", baseURL, "/denylist", t, 200) | ||
| // check both permutations, in case the server reordered them | ||
| if !reflect.DeepEqual(data, []interface{}{"abc", "def"}) && !reflect.DeepEqual(data, []interface{}{"def", "abc"}) { | ||
| t.Fatalf("Expected doubleton from GET, but got %#v", data) | ||
| } | ||
|
|
||
| otr.Stop() | ||
| time.Sleep(3 * time.Second) | ||
| otr.Start() | ||
|
|
||
| time.Sleep(3 * time.Second) | ||
|
|
||
| // denylist should have persisted across the restart | ||
|
|
||
| // GET list with both rules | ||
| data = helpers.DoRequest("GET", baseURL, "/denylist", t, 200) | ||
| // check both permutations, in case the server reordered them | ||
| if !reflect.DeepEqual(data, []interface{}{"abc", "def"}) && !reflect.DeepEqual(data, []interface{}{"def", "abc"}) { | ||
| t.Fatalf("Expected doubleton from GET, but got %#v", data) | ||
| } | ||
|
|
||
| // denylist should still be modifiable | ||
|
|
||
| // DELETE first rule | ||
| helpers.DoRequest("DELETE", baseURL, "/denylist/abc", t, 204) | ||
| // GET list with only second rule | ||
| data = helpers.DoRequest("GET", baseURL, "/denylist", t, 200) | ||
| if !reflect.DeepEqual(data, []interface{}{"def"}) { | ||
| t.Fatalf("Expected singleton from GET, but got %#V", data) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| package harness | ||
|
|
||
| import ( | ||
| "os/exec" | ||
| ) | ||
|
|
||
| type PostgresServer struct { | ||
| ConnStr string | ||
| } | ||
|
|
||
| func runCommandWithLogs(name string, args ...string) { | ||
| cmd := exec.Command(name, args...) | ||
| cmd.Stderr = makeLogStreamer("postgres", "stderr") | ||
| cmd.Stdout = makeLogStreamer("postgres", "stdout") | ||
| err := cmd.Start() | ||
| if err != nil { | ||
| panic("Error starting up postgres: " + err.Error()) | ||
| } | ||
| } | ||
|
|
||
| func StartPostgresServer() *PostgresServer { | ||
| runCommandWithLogs( | ||
| "pg_ctlcluster", | ||
| "11", | ||
| "main", | ||
| "start", | ||
| ) | ||
|
|
||
| waitTCP("127.0.0.1:5432") | ||
|
|
||
| runCommandWithLogs( | ||
| "runuser", | ||
| "-u", | ||
| "postgres", | ||
| "--", | ||
| "psql", | ||
| "-c", | ||
| "ALTER USER postgres WITH PASSWORD 'postgres';", | ||
| ) | ||
|
|
||
| return &PostgresServer{ | ||
| ConnStr: "postgres://postgres:postgres@localhost/postgres", | ||
| } | ||
| } | ||
|
|
||
| func (server *PostgresServer) Stop() { | ||
| runCommandWithLogs( | ||
| "pg_ctlcluster", | ||
| "11", | ||
| "main", | ||
| "stop", | ||
| ) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| package helpers | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "encoding/json" | ||
| "io" | ||
| "net/http" | ||
| "testing" | ||
| ) | ||
|
|
||
| func DoRequest(method string, baseURL string, path string, t *testing.T, expectedCode int) interface{} { | ||
| req, err := http.NewRequest(method, baseURL+path, &bytes.Buffer{}) | ||
| if err != nil { | ||
| t.Fatalf("Error creating req: %s", err) | ||
| } | ||
| req.Header.Set("Content-Type", "application/json") | ||
| resp, err := (&http.Client{}).Do(req) | ||
| if err != nil { | ||
| t.Fatalf("Error sending request: %s", err) | ||
| } | ||
|
|
||
| defer resp.Body.Close() | ||
|
|
||
| respBody, err := io.ReadAll(resp.Body) | ||
| if err != nil { | ||
| t.Fatalf("Error eceiving response body: %s", err) | ||
| } | ||
|
|
||
| if resp.StatusCode != expectedCode { | ||
| t.Fatalf("Expected status code %d, but got %d.\nBody was: %s", expectedCode, resp.StatusCode, respBody) | ||
| } | ||
|
|
||
| if expectedCode == 200 { | ||
| var data interface{} | ||
| err = json.Unmarshal(respBody, &data) | ||
| if err != nil { | ||
| t.Fatalf("Error parsing JSON response: %s", err) | ||
| } | ||
|
|
||
| return data | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.