Run with AWS

Phalanx supports Amazon S3 and etcd.

If you want to try it on your local machine, you can start LocalStack and etcd with the following command to prepare a pseudo AWS environment.

% docker-compose up --force-recreate etcd etcdkeeper localstack

Start Phalanx cluster with DynamoDB metastore

Start the first node:

% ./bin/phalanx --index-metastore-uri=etcd://phalanx-metadata

Start the second node:

% ./bin/phalanx --index-metastore-uri=etcd://phalanx-metadata --bind-port=2001 --grpc-port=5001 --http-port=8001 --seed-addresses=localhost:2000

Start the third node:

% ./bin/phalanx --index-metastore-uri=etcd://phalanx-metadata --bind-port=2002 --grpc-port=5002 --http-port=8002 --seed-addresses=localhost:2000

Use the following command to create an index of 6 shards. If you need more nodes, start them in the same way as in the above command.

Create index with S3 and DyunamoDB

Use S3 as index storage, and create a lock on DynamoDB to avoid write conflicts.

% curl -XPUT -H 'Content-type: application/json' http://localhost:8000/v1/indexes/example --data-binary '
{
    "index_uri": "s3://phalanx-indexes/example",
    "lock_uri": "dynamodb://phalanx-locks/example",
    "index_mapping": {
        "id": {
            "type": "numeric",
            "options": {
                "index": true,
                "store": true,
                "sortable": true,
                "aggregatable": true
            }
        },
        "text": {
            "type": "text",
            "options": {
                "index": true,
                "store": true,
                "term_positions": true,
                "highlight": true,
                "sortable": true,
                "aggregatable": true
            },
            "analyzer": {
                "char_filters": [
                    {
                        "name": "ascii_folding"
                    },
                    {
                        "name": "unicode_normalize",
                        "options": {
                            "form": "NFKC"
                        }
                    }
                ],
                "tokenizer": {
                    "name": "unicode"
                },
                "token_filters": [
                    {
                        "name": "lower_case"
                    }
                ]
            }
        }
    },
    "num_shards": 6,
    "default_search_field": "_all",
    "default_analyzer": {
        "tokenizer": {
            "name": "unicode"
        },
        "token_filters": [
            {
                "name": "lower_case"
            }
        ]
    }
}
'

Cluster status

% curl -XGET http://localhost:8000/cluster | jq .
{
  "indexer_assignment": {
    "example": {
      "shard-0kZFXhrZ": "node-8giCE4QA",
      "shard-FretswBz": "node-Rh6kmVO8",
      "shard-Lf7rlqwd": "node-Rh6kmVO8",
      "shard-WvUd7lWm": "node-Rh6kmVO8",
      "shard-f4f6jbCi": "node-Rh6kmVO8",
      "shard-kCPFctnO": "node-8giCE4QA"
    }
  },
  "indexes": {
    "example": {
      "index_lock_uri": "etcd://phalanx-locks/example",
      "index_uri": "minio://phalanx-indexes/example",
      "shards": {
        "shard-0kZFXhrZ": {
          "shard_lock_uri": "etcd://phalanx-locks/example/shard-0kZFXhrZ",
          "shard_uri": "minio://phalanx-indexes/example/shard-0kZFXhrZ"
        },
        "shard-FretswBz": {
          "shard_lock_uri": "etcd://phalanx-locks/example/shard-FretswBz",
          "shard_uri": "minio://phalanx-indexes/example/shard-FretswBz"
        },
        "shard-Lf7rlqwd": {
          "shard_lock_uri": "etcd://phalanx-locks/example/shard-Lf7rlqwd",
          "shard_uri": "minio://phalanx-indexes/example/shard-Lf7rlqwd"
        },
        "shard-WvUd7lWm": {
          "shard_lock_uri": "etcd://phalanx-locks/example/shard-WvUd7lWm",
          "shard_uri": "minio://phalanx-indexes/example/shard-WvUd7lWm"
        },
        "shard-f4f6jbCi": {
          "shard_lock_uri": "etcd://phalanx-locks/example/shard-f4f6jbCi",
          "shard_uri": "minio://phalanx-indexes/example/shard-f4f6jbCi"
        },
        "shard-kCPFctnO": {
          "shard_lock_uri": "etcd://phalanx-locks/example/shard-kCPFctnO",
          "shard_uri": "minio://phalanx-indexes/example/shard-kCPFctnO"
        }
      }
    }
  },
  "nodes": {
    "node-09upDjwO": {
      "addr": "0.0.0.0",
      "meta": {
        "grpc_port": 5001,
        "http_port": 8001,
        "roles": [
          "indexer",
          "searcher"
        ]
      },
      "port": 2001,
      "state": "alive"
    },
    "node-8giCE4QA": {
      "addr": "0.0.0.0",
      "meta": {
        "grpc_port": 5002,
        "http_port": 8002,
        "roles": [
          "indexer",
          "searcher"
        ]
      },
      "port": 2002,
      "state": "alive"
    },
    "node-Rh6kmVO8": {
      "addr": "0.0.0.0",
      "meta": {
        "grpc_port": 5000,
        "http_port": 8000,
        "roles": [
          "indexer",
          "searcher"
        ]
      },
      "port": 2000,
      "state": "alive"
    }
  },
  "searcher_assignment": {
    "example": {
      "shard-0kZFXhrZ": [
        "node-8giCE4QA",
        "node-Rh6kmVO8",
        "node-09upDjwO"
      ],
      "shard-FretswBz": [
        "node-Rh6kmVO8",
        "node-09upDjwO",
        "node-8giCE4QA"
      ],
      "shard-Lf7rlqwd": [
        "node-Rh6kmVO8",
        "node-09upDjwO",
        "node-8giCE4QA"
      ],
      "shard-WvUd7lWm": [
        "node-Rh6kmVO8",
        "node-8giCE4QA",
        "node-09upDjwO"
      ],
      "shard-f4f6jbCi": [
        "node-Rh6kmVO8",
        "node-8giCE4QA",
        "node-09upDjwO"
      ],
      "shard-kCPFctnO": [
        "node-8giCE4QA",
        "node-Rh6kmVO8",
        "node-09upDjwO"
      ]
    }
  }
}

Add / Update documents

% curl -XPUT -H 'Content-type: application/x-ndjson' http://localhost:8000/v1/indexes/example/documents --data-binary '
{"_id":"1", "id":1, "text":"This is an example document 1."}
{"_id":"2", "id":2, "text":"This is an example document 2."}
{"_id":"3", "id":3, "text":"This is an example document 3."}
{"_id":"4", "id":4, "text":"This is an example document 4."}
{"_id":"5", "id":5, "text":"This is an example document 5."}
{"_id":"6", "id":6, "text":"This is an example document 6."}
{"_id":"7", "id":7, "text":"This is an example document 7."}
{"_id":"8", "id":8, "text":"This is an example document 8."}
{"_id":"9", "id":9, "text":"This is an example document 9."}
{"_id":"10", "id":10, "text":"This is an example document 10."}
{"_id":"11", "id":11, "text":"This is an example document 11."}
{"_id":"12", "id":12, "text":"This is an example document 12."}
'

Delete documents

% curl -XDELETE -H 'Content-type: text/plain' http://localhost:8000/v1/indexes/example/documents --data-binary '
1
2
3
4
5
6
7
8
9
10
11
12
'
% curl -XPOST -H 'Content-type: application/json' http://localhost:8000/v1/indexes/example/_search --data-binary '
{
    "query": {
        "type": "boolean",
        "options": {
            "must": [
                {
                    "type": "query_string",
                    "options": {
                        "query": "*"
                    }
                }
            ],
            "min_should": 1,
            "boost": 1.0
        }
    },
    "start": 0,
    "num": 10,
    "sort_by": "-_score",
    "fields": [
        "id",
        "text"
    ],
    "aggregations": {
        "timestamp_date_range": {
            "type": "date_range",
            "options": {
                "field": "_timestamp",
                "ranges": {
                    "year_before_last": {
                        "start": "2020-01-01T00:00:00Z",
                        "end": "2021-01-01T00:00:00Z"
                    },
                    "last_year": {
                        "start": "2021-01-01T00:00:00Z",
                        "end": "2022-01-01T00:00:00Z"
                    },
                    "this_year": {
                        "start": "2022-01-01T00:00:00Z",
                        "end": "2023-01-01T00:00:00Z"
                    }
                }
            }
        }
    }
}
' | jq .
{
  "aggregations": {
    "timestamp_date_range": {
      "last_year": 0,
      "this_year": 12,
      "year_before_last": 0
    }
  },
  "documents": [
    {
      "fields": {
        "id": 1,
        "text": "This is an example document 1."
      },
      "id": "1",
      "score": 0.08287343490634301,
      "timestamp": 1643177200809294300
    },
    {
      "fields": {
        "id": 2,
        "text": "This is an example document 2."
      },
      "id": "2",
      "score": 0.08287343490634301,
      "timestamp": 1643177200809316000
    },
    {
      "fields": {
        "id": 4,
        "text": "This is an example document 4."
      },
      "id": "4",
      "score": 0.047891143480830164,
      "timestamp": 1643177200809422600
    },
    {
      "fields": {
        "id": 7,
        "text": "This is an example document 7."
      },
      "id": "7",
      "score": 0.047891143480830164,
      "timestamp": 1643177200809437000
    },
    {
      "fields": {
        "id": 10,
        "text": "This is an example document 10."
      },
      "id": "10",
      "score": 0.047891143480830164,
      "timestamp": 1643177200809448400
    },
    {
      "fields": {
        "id": 11,
        "text": "This is an example document 11."
      },
      "id": "11",
      "score": 0.047891143480830164,
      "timestamp": 1643177200809457700
    },
    {
      "fields": {
        "id": 5,
        "text": "This is an example document 5."
      },
      "id": "5",
      "score": 0.06069608755660118,
      "timestamp": 1643177200809490000
    },
    {
      "fields": {
        "id": 6,
        "text": "This is an example document 6."
      },
      "id": "6",
      "score": 0.06069608755660118,
      "timestamp": 1643177200809517000
    },
    {
      "fields": {
        "id": 12,
        "text": "This is an example document 12."
      },
      "id": "12",
      "score": 0.06069608755660118,
      "timestamp": 1643177200809526300
    },
    {
      "fields": {
        "id": 3,
        "text": "This is an example document 3."
      },
      "id": "3",
      "score": 0.06069608755660118,
      "timestamp": 1643177200809375200
    }
  ],
  "hits": 12,
  "index_name": "example"
}

Delete index

% curl -XDELETE http://localhost:8000/v1/indexes/example

results matching ""

    No results matching ""