Run with MinIO and etcd
To experience Phalanx functionality, let's start Phalanx with MinIO and etcd. This repository has a docker-compose.yml file. With it, you can easily launch MinIO and etcd on Docker.
% docker-compose up --force-recreate etcd etcdkeeper minio
Once the container has been started, you can check the MinIO and etcd data in your browser at the following URL.
ETCD Keeper
http://localhost:8080/etcdkeeper/
Start Phalanx with etcd metastore
Start the first node:
% ./bin/phalanx --index-metastore-uri=etcd://phalanx-metadata
Start the second node:
% ./bin/phalanx --index-metastore-uri=etcd://phalanx-metadata --bind-port=2001 --grpc-port=5001 --http-port=8001 --seed-addresses=localhost:2000
Start the third node:
% ./bin/phalanx --index-metastore-uri=etcd://phalanx-metadata --bind-port=2002 --grpc-port=5002 --http-port=8002 --seed-addresses=localhost:2000
Use the following command to create an index of 6 shards. If you need more nodes, start them in the same way as in the above command.
Create index with MinIO and etcd
Use MinIO as index storage, and create a lock on etcd to avoid write conflicts.
% curl -XPUT -H 'Content-type: application/json' http://localhost:8000/v1/indexes/example --data-binary '
{
"index_uri": "minio://phalanx-indexes/example",
"lock_uri": "etcd://phalanx-locks/example",
"index_mapping": {
"id": {
"type": "numeric",
"options": {
"index": true,
"store": true,
"sortable": true,
"aggregatable": true
}
},
"text": {
"type": "text",
"options": {
"index": true,
"store": true,
"term_positions": true,
"highlight": true,
"sortable": true,
"aggregatable": true
},
"analyzer": {
"char_filters": [
{
"name": "ascii_folding"
},
{
"name": "unicode_normalize",
"options": {
"form": "NFKC"
}
}
],
"tokenizer": {
"name": "unicode"
},
"token_filters": [
{
"name": "lower_case"
}
]
}
}
},
"num_shards": 6,
"default_search_field": "_all",
"default_analyzer": {
"tokenizer": {
"name": "unicode"
},
"token_filters": [
{
"name": "lower_case"
}
]
}
}
'
Cluster status
% curl -XGET http://localhost:8000/cluster | jq .
{
"indexer_assignment": {
"example": {
"shard-0kZFXhrZ": "node-8giCE4QA",
"shard-FretswBz": "node-Rh6kmVO8",
"shard-Lf7rlqwd": "node-Rh6kmVO8",
"shard-WvUd7lWm": "node-Rh6kmVO8",
"shard-f4f6jbCi": "node-Rh6kmVO8",
"shard-kCPFctnO": "node-8giCE4QA"
}
},
"indexes": {
"example": {
"index_lock_uri": "etcd://phalanx-locks/example",
"index_uri": "minio://phalanx-indexes/example",
"shards": {
"shard-0kZFXhrZ": {
"shard_lock_uri": "etcd://phalanx-locks/example/shard-0kZFXhrZ",
"shard_uri": "minio://phalanx-indexes/example/shard-0kZFXhrZ"
},
"shard-FretswBz": {
"shard_lock_uri": "etcd://phalanx-locks/example/shard-FretswBz",
"shard_uri": "minio://phalanx-indexes/example/shard-FretswBz"
},
"shard-Lf7rlqwd": {
"shard_lock_uri": "etcd://phalanx-locks/example/shard-Lf7rlqwd",
"shard_uri": "minio://phalanx-indexes/example/shard-Lf7rlqwd"
},
"shard-WvUd7lWm": {
"shard_lock_uri": "etcd://phalanx-locks/example/shard-WvUd7lWm",
"shard_uri": "minio://phalanx-indexes/example/shard-WvUd7lWm"
},
"shard-f4f6jbCi": {
"shard_lock_uri": "etcd://phalanx-locks/example/shard-f4f6jbCi",
"shard_uri": "minio://phalanx-indexes/example/shard-f4f6jbCi"
},
"shard-kCPFctnO": {
"shard_lock_uri": "etcd://phalanx-locks/example/shard-kCPFctnO",
"shard_uri": "minio://phalanx-indexes/example/shard-kCPFctnO"
}
}
}
},
"nodes": {
"node-09upDjwO": {
"addr": "0.0.0.0",
"meta": {
"grpc_port": 5001,
"http_port": 8001,
"roles": [
"indexer",
"searcher"
]
},
"port": 2001,
"state": "alive"
},
"node-8giCE4QA": {
"addr": "0.0.0.0",
"meta": {
"grpc_port": 5002,
"http_port": 8002,
"roles": [
"indexer",
"searcher"
]
},
"port": 2002,
"state": "alive"
},
"node-Rh6kmVO8": {
"addr": "0.0.0.0",
"meta": {
"grpc_port": 5000,
"http_port": 8000,
"roles": [
"indexer",
"searcher"
]
},
"port": 2000,
"state": "alive"
}
},
"searcher_assignment": {
"example": {
"shard-0kZFXhrZ": [
"node-8giCE4QA",
"node-Rh6kmVO8",
"node-09upDjwO"
],
"shard-FretswBz": [
"node-Rh6kmVO8",
"node-09upDjwO",
"node-8giCE4QA"
],
"shard-Lf7rlqwd": [
"node-Rh6kmVO8",
"node-09upDjwO",
"node-8giCE4QA"
],
"shard-WvUd7lWm": [
"node-Rh6kmVO8",
"node-8giCE4QA",
"node-09upDjwO"
],
"shard-f4f6jbCi": [
"node-Rh6kmVO8",
"node-8giCE4QA",
"node-09upDjwO"
],
"shard-kCPFctnO": [
"node-8giCE4QA",
"node-Rh6kmVO8",
"node-09upDjwO"
]
}
}
}
Add / Update documents
% curl -XPUT -H 'Content-type: application/x-ndjson' http://localhost:8000/v1/indexes/example/documents --data-binary '
{"_id":"1", "id":1, "text":"This is an example document 1."}
{"_id":"2", "id":2, "text":"This is an example document 2."}
{"_id":"3", "id":3, "text":"This is an example document 3."}
{"_id":"4", "id":4, "text":"This is an example document 4."}
{"_id":"5", "id":5, "text":"This is an example document 5."}
{"_id":"6", "id":6, "text":"This is an example document 6."}
{"_id":"7", "id":7, "text":"This is an example document 7."}
{"_id":"8", "id":8, "text":"This is an example document 8."}
{"_id":"9", "id":9, "text":"This is an example document 9."}
{"_id":"10", "id":10, "text":"This is an example document 10."}
{"_id":"11", "id":11, "text":"This is an example document 11."}
{"_id":"12", "id":12, "text":"This is an example document 12."}
'
Delete documents
% curl -XDELETE -H 'Content-type: text/plain' http://localhost:8000/v1/indexes/example/documents --data-binary '
1
2
3
4
5
6
7
8
9
10
11
12
'
Search
% curl -XPOST -H 'Content-type: application/json' http://localhost:8000/v1/indexes/example/_search --data-binary '
{
"query": {
"type": "boolean",
"options": {
"must": [
{
"type": "query_string",
"options": {
"query": "*"
}
}
],
"min_should": 1,
"boost": 1.0
}
},
"start": 0,
"num": 10,
"sort_by": "-_score",
"fields": [
"id",
"text"
],
"aggregations": {
"timestamp_date_range": {
"type": "date_range",
"options": {
"field": "_timestamp",
"ranges": {
"year_before_last": {
"start": "2020-01-01T00:00:00Z",
"end": "2021-01-01T00:00:00Z"
},
"last_year": {
"start": "2021-01-01T00:00:00Z",
"end": "2022-01-01T00:00:00Z"
},
"this_year": {
"start": "2022-01-01T00:00:00Z",
"end": "2023-01-01T00:00:00Z"
}
}
}
}
}
}
' | jq .
{
"aggregations": {
"timestamp_date_range": {
"last_year": 0,
"this_year": 12,
"year_before_last": 0
}
},
"documents": [
{
"fields": {
"id": 1,
"text": "This is an example document 1."
},
"id": "1",
"score": 0.08287343490634301,
"timestamp": 1643177200809294300
},
{
"fields": {
"id": 2,
"text": "This is an example document 2."
},
"id": "2",
"score": 0.08287343490634301,
"timestamp": 1643177200809316000
},
{
"fields": {
"id": 4,
"text": "This is an example document 4."
},
"id": "4",
"score": 0.047891143480830164,
"timestamp": 1643177200809422600
},
{
"fields": {
"id": 7,
"text": "This is an example document 7."
},
"id": "7",
"score": 0.047891143480830164,
"timestamp": 1643177200809437000
},
{
"fields": {
"id": 10,
"text": "This is an example document 10."
},
"id": "10",
"score": 0.047891143480830164,
"timestamp": 1643177200809448400
},
{
"fields": {
"id": 11,
"text": "This is an example document 11."
},
"id": "11",
"score": 0.047891143480830164,
"timestamp": 1643177200809457700
},
{
"fields": {
"id": 5,
"text": "This is an example document 5."
},
"id": "5",
"score": 0.06069608755660118,
"timestamp": 1643177200809490000
},
{
"fields": {
"id": 6,
"text": "This is an example document 6."
},
"id": "6",
"score": 0.06069608755660118,
"timestamp": 1643177200809517000
},
{
"fields": {
"id": 12,
"text": "This is an example document 12."
},
"id": "12",
"score": 0.06069608755660118,
"timestamp": 1643177200809526300
},
{
"fields": {
"id": 3,
"text": "This is an example document 3."
},
"id": "3",
"score": 0.06069608755660118,
"timestamp": 1643177200809375200
}
],
"hits": 12,
"index_name": "example"
}
Delete index
% curl -XDELETE http://localhost:8000/v1/indexes/example