What is Elasticsearch?

The Overview:

Hostname: es-master-1, Private IP: 172.31.0.77 
Hostname: es-master-2, Private IP: 172.31.0.45
Hostname: es-master-3, Private IP: 172.31.1.31
Hostname: es-data-1, Private IP:172.31.2.30 
Hostname: es-data-2, Private IP:172.31.0.83
es-data-1: 10GB assigned to /dev/vdb
es-data-2: 10GB assigned to /dev/vdb

Setup the Elasticsearch Master Nodes

$ cat > /etc/hosts << EOF
127.0.0.1 localhost
172.31.0.77 es-master-1
172.31.0.45 es-master-2
172.31.1.31 es-master-3
172.31.2.30 es-data-1
172.31.0.83 es-data-2
EOF
$ apt update && apt upgrade -y
$ apt install software-properties-common python-software-properties apt-transport-https -y
$ wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
$ echo "deb https://artifacts.elastic.co/packages/6.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-6.x.list
$ apt update
$ apt install default-jdk -y
$ apt install elasticsearch -y
# example of log splitting:
...
path:
logs: /var/log/elasticsearch
data: /var/data/elasticsearch
...
# example of data paths:
...
path:
data:
- /mnt/elasticsearch_1
- /mnt/elasticsearch_2
- /mnt/elasticsearch_3
...
$ cat > /etc/elasticsearch/elasticsearch.yml << EOF
cluster.name: es-cluster
node.name: \${HOSTNAME}
node.master: true
node.data: false
path.logs: /var/log/elasticsearch
bootstrap.memory_lock: true
network.host: 0.0.0.0
discovery.zen.minimum_master_nodes: 2
discovery.zen.ping.unicast.hosts: ["es-master-1", "es-master-2", "es-master-3"]
EOF
$ cat > /etc/default/elasticsearch << EOF
ES_STARTUP_SLEEP_TIME=5
MAX_OPEN_FILES=65536
MAX_LOCKED_MEMORY=unlimited
EOF
$ vim /usr/lib/systemd/system/elasticsearch.service

[Service]
LimitMEMLOCK=infinity
LimitNOFILE=65535
LimitNPROC=4096
...
$ cat > /etc/security/limits.conf << EOF
elasticsearch soft memlock unlimited
elasticsearch hard memlock unlimited
EOF
$ sysctl -w vm.max_map_count=262144
$ mkdir /usr/share/elasticsearch/data
$ chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/data
$ systemctl daemon-reload
$ systemctl enable elasticsearch
$ systemctl restart elasticsearch
$ curl http://127.0.0.1:9200/_cluster/health?pretty
{
"cluster_name" : "es-cluster",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 3,
"number_of_data_nodes" : 0,
"active_primary_shards" : 0,
"active_shards" : 0,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}
$ curl http://127.0.0.1:9200/_cat/nodes?v
ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
10.163.68.8 11 80 18 0.28 0.14 0.09 mi - es-master-2
10.163.68.5 14 80 14 0.27 0.18 0.11 mi * es-master-1
10.163.68.4 15 79 6 0.62 0.47 0.18 mi - es-master-3

Setup the Elasticsearch Data Nodes

$ cat > /etc/hosts << EOF
127.0.0.1 localhost
172.31.0.77 es-master-1
172.31.0.45 es-master-2
172.31.1.31 es-master-3
172.31.2.30 es-data-1
172.31.0.83 es-data-2
EOF
$ apt update && apt upgrade -y
$ apt install software-properties-common python-software-properties apt-transport-https -y
$ wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
$ echo "deb https://artifacts.elastic.co/packages/6.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-6.x.list
$ apt update
$ apt install default-jdk -y
$ apt install elasticsearch -y
$ lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
vda 253:0 0 25G 0 disk
└─vda1 253:1 0 25G 0 part /
vdb 253:16 0 10G 0 disk <----
$ mkfs.xfs /dev/vdb
$ mkdir /data
$ mkdir /data/nodes
$ chown -R elasticsearch:elasticsearch /data
$ chown -R elasticsearch:elasticsearch /data/nodes
$ echo '/dev/vdb /data xfs defaults 0 0' >> /etc/fstab
$ mount -a
$ df -h
Filesystem Size Used Avail Use% Mounted on
udev 994M 0 994M 0% /dev
tmpfs 201M 3.1M 197M 2% /run
/dev/vda1 25G 1.8G 23G 8% /
/dev/vdb 10G 33M 10G 1% /data
$ cat > /etc/elasticsearch/elasticsearch.yml << EOF
cluster.name: es-cluster
node.name: \${HOSTNAME}
node.master: false
node.data: true
path.data: /data
path.logs: /var/log/elasticsearch
bootstrap.memory_lock: true
network.host: 0.0.0.0
discovery.zen.minimum_master_nodes: 2
discovery.zen.ping.unicast.hosts: ["es-master-1", "es-master-2", "es-master-3"]
EOF
$ cat > /etc/default/elasticsearch << EOF
ES_STARTUP_SLEEP_TIME=5
MAX_OPEN_FILES=65536
MAX_LOCKED_MEMORY=unlimited
EOF
$ vim /usr/lib/systemd/system/elasticsearch.service
[Service]
LimitMEMLOCK=infinity
LimitNOFILE=65535
LimitNPROC=4096
$ cat > /etc/security/limits.conf << EOF
elasticsearch soft memlock unlimited
elasticsearch hard memlock unlimited
EOF
$ systemctl daemon-reload
$ systemctl enable elasticsearch
$ systemctl restart elasticsearch
$ curl http://127.0.0.1:9200/_cluster/health?pretty
{
"cluster_name" : "es-cluster",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 5,
"number_of_data_nodes" : 2,
"active_primary_shards" : 0,
"active_shards" : 0,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}
$ curl http://127.0.0.1:9200/_cat/nodes?v
ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
10.163.68.7 9 96 6 0.12 0.11 0.03 di - es-data-2
10.163.68.5 10 80 2 0.20 0.09 0.08 mi * es-master-1
10.163.68.11 12 96 9 0.12 0.09 0.03 di - es-data-1
10.163.68.4 10 79 0 0.00 0.12 0.11 mi - es-master-3
10.163.68.8 12 79 1 0.05 0.06 0.07 mi - es-master-2

Interact with Elasticsearch

$ curl http://127.0.0.1:9200
{
"name" : "es-data-1",
"cluster_name" : "es-cluster",
"cluster_uuid" : "5BLs4sxsSEK-4OxlGnmlmw",
"version" : {
"number" : "6.7.0",
"build_flavor" : "default",
"build_type" : "deb",
"build_hash" : "8453f77",
"build_date" : "2019-03-21T15:32:29.844721Z",
"build_snapshot" : false,
"lucene_version" : "7.7.0",
"minimum_wire_compatibility_version" : "5.6.0",
"minimum_index_compatibility_version" : "5.0.0"
},
"tagline" : "You Know, for Search"
}
$ curl http://127.0.0.1:9200/_cat/health?v
epoch timestamp cluster status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent
1554154652 21:37:32 es-cluster green 5 2 10 5 0 0 0 0 - 100.0%
$ curl -H 'Content-Type: application/json' -XPOST http://127.0.0.1:9200/first-index/docs/ -d '
{"username": "mikes", "name": "mike", "surname": "steyn", "location": {"country": "south africa", "city": "cape town"}, "hobbies": ["sport", "coffee"]}'
$ curl -H 'Content-Type: application/json' -XPOST http://127.0.0.1:9200/first-index/docs/ -d '
{"username": "clarissas", "name": "clarissa", "surname": "smith", "location": {"country": "ireland", "city": "dublin"}, "hobbies": ["shopping", "reading", "chess"]}'

$ curl -H 'Content-Type: application/json' -XPOST http://127.0.0.1:9200/first-index/docs/ -d '
{"username": "franka", "name": "frank", "surname": "adams", "location": {"country": "new zealand", "city": "auckland"}, "hobbies": ["programming", "swimming", "rugby"]}'
$ curl http://127.0.0.1:9200/_cat/indices?v
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
green open first-index 1o6yM7tCSqagqoeihKM7_g 5 1 3 0 40.6kb 20.3kb
$ curl http://127.0.0.1:9200/first-index/_search?pretty
{
"took" : 116,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" : [
{
"_index" : "first-index",
"_type" : "docs",
"_id" : "-NTO2mkB8pugP4aC2jtZ",
"_score" : 1.0,
"_source" : {
"username" : "mikes",
"name" : "mike",
"surname" : "steyn",
"location" : {
"country" : "south africa",
"city" : "cape town"
},
"hobbies" : [
"sport",
"coffee"
]
}
},
{
"_index" : "first-index",
"_type" : "docs",
"_id" : "-tTR2mkB8pugP4aCAzvG",
"_score" : 1.0,
"_source" : {
"username" : "franka",
"name" : "frank",
"surname" : "adams",
"location" : {
"country" : "new zealand",
"city" : "auckland"
},
"hobbies" : [
"programming",
"swimming",
"rugby"
]
}
},
{
"_index" : "first-index",
"_type" : "docs",
"_id" : "-dTP2mkB8pugP4aC1ztI",
"_score" : 1.0,
"_source" : {
"username" : "clarissas",
"name" : "clarissa",
"surname" : "smith",
"location" : {
"country" : "ireland",
"city" : "dublin"
},
"hobbies" : [
"shopping",
"reading",
"chess"
]
}
}
]
}
}
$ curl http://127.0.0.1:9200/_cat/shards?v
index shard prirep state docs store ip node
first-index 4 p STARTED 0 230b 10.163.68.7 es-data-2
first-index 4 r STARTED 0 230b 10.163.68.11 es-data-1
first-index 2 p STARTED 0 230b 10.163.68.7 es-data-2
first-index 2 r STARTED 0 230b 10.163.68.11 es-data-1
first-index 3 r STARTED 1 6.6kb 10.163.68.7 es-data-2
first-index 3 p STARTED 1 6.6kb 10.163.68.11 es-data-1
first-index 1 r STARTED 2 13kb 10.163.68.7 es-data-2
first-index 1 p STARTED 2 13kb 10.163.68.11 es-data-1
first-index 0 p STARTED 0 230b 10.163.68.7 es-data-2
first-index 0 r STARTED 0 230b 10.163.68.11 es-data-1
$ curl http://127.0.0.1:9200/_cat/allocation?v
shards disk.indices disk.used disk.avail disk.total disk.percent host ip node
5 20.3kb 32.4mb 9.9gb 9.9gb 0 10.163.68.11 10.163.68.11 es-data-1
5 20.3kb 32.4mb 9.9gb 9.9gb 0 10.163.68.7 10.163.68.7 es-data-2
$ curl -s http://127.0.0.1:9200/first-index/_search?q=surname=smith | jq .
{
"took": 22,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 0.2876821,
"hits": [
{
"_index": "first-index",
"_type": "docs",
"_id": "-dTP2mkB8pugP4aC1ztI",
"_score": 0.2876821,
"_source": {
"username": "clarissas",
"name": "clarissa",
"surname": "smith",
"location": {
"country": "ireland",
"city": "dublin"
},
"hobbies": [
"shopping",
"reading",
"chess"
]
}
}
]
}
}
$ curl -s http://127.0.0.1:9200/first-index/_search?q=hobbies=rugby | jq .
{
"took": 23,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 0.64072424,
"hits": [
{
"_index": "first-index",
"_type": "docs",
"_id": "-tTR2mkB8pugP4aCAzvG",
"_score": 0.64072424,
"_source": {
"username": "franka",
"name": "frank",
"surname": "adams",
"location": {
"country": "new zealand",
"city": "auckland"
},
"hobbies": [
"programming",
"swimming",
"rugby"
]
}
}
]
}
}

More on Elasticsearch:

Resources

--

--

--

DevOps Engineer and Open Source Enthusiast

Love podcasts or audiobooks? Learn on the go with our new app.

Recommended from Medium

CS 371g Summer 2020: Chengzhi Han

Azure Purview — Metadata Management and Data Governance Service

Using GitHub with Unity

How to save time while working as a computer engineer.

Terraform Weekly — Issue #13

Fill in missing Date ranges using MySQL

Why AWS is more cost effective than traditional data centres

RIP Test: My 3-Step Process for Solving Any Technical Problem

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store
Ruan Bekker

Ruan Bekker

DevOps Engineer and Open Source Enthusiast

More from Medium

Elasticsearch REST Java Client

Creating a Kafka sandbox in Docker

Connecting Go application to Oracle Database-On Prem and Autonomous , with and without Oracle…

Java-Spring Boot | GraphQL | Docker