Integrate AWS Elastic Search with AWS Lambda using Nodejs

Posted By : Ankit Uniyal | 30-Jan-2018

nodejs

Hi,

In this blog, we will cover up how we can load S3 content into Elastic Search domain using Nodejs. We need to follow the below steps in order to loading S3 streaming content into Elastic Search domain.

Below are the steps to be follow :

1.Get S3 bucket object and then we need to check whether an index is already created on Elastic Search or not.

var getSearchMediaItemsFromS3Bucket = function(event, next, callback, results) {
    console.log("getSearchMediaItemsFromS3Bucket function started");
    var s3 = new AWS.S3();
    var params = {
        Bucket: configurationHolder.config.bucketName,
        Key: configurationHolder.config.s3UploadFilePath
    }
    s3.getObject(params, function(err, data) {
        if (err) {
            console.log("Error in getting CSV file from S3 bucket", err);
        } else {
            console.log("Content is", data);

            var dataObject = JSON.parse(data.Body);
            console.log("dataObject is to be", dataObject);
            next(null, dataObject);
        }
    })
}

2.Check if index name already exists in Elastic Search or not.

var checkIfIndexAlreadyExists = function(next, indexName, results) {
    console.log("checkIfIndexAlreadyExists fucntion started", indexName);
    var elasticSearch = require('elasticsearch');
    var elasticSearchClient = new elasticSearch.Client({
        accessKeyId: 'your_keyid',
        secretAccessKey: 'your_secret',
        service: 'es',
        region: 'your_region',
        host: 'your_end_point'
    });
    var isIndexNameExists = false;
    elasticSearchClient.indices.exists({
        index: indexName
    }, function(err, data) {
        if (err) {
            console.log("Index doesn't exists");
            next(null, {
                isIndexNameExists: isIndexNameExists
            });
        } else {
            console.log("Index exists", data);
            isIndexNameExists = true;
            next(null, {
                isIndexNameExists: isIndexNameExists
            });
        }
    });
}

3.If an index is already present with the provided name then we need to delete the previous index.

var deleteIndexIfAlreadyExists = function(next, isIndexNameExists, indexName, results) {
    console.log("deleteIndexIfAlreadyExists function started", indexName);
    var elasticSearch = require('elasticsearch');
    var elasticSearchClient = new elasticSearch.Client({
        accessKeyId: 'your_keyid',
        secretAccessKey: 'your_secret',
        service: 'es',
        region: 'your_region',
        host: 'your_end_point'
    });
    if (isIndexNameExists == true || isIndexNameExists == 'true') {
        next(null, true);
    } else {
        elasticSearchClient.indices.delete({
            index: indexName,
            timeout: '60000s'
        }, function(err, data) {
            if (err) {
                next(err);
            } else {
                console.log("data reply received", data);
                next(null, true);
            }
        });
    }
}

4.Then, We will create an index on Elastic Search using Nodejs :

var createIndexAndStreamDataOnES = function(next, elasticSearchContent, indexName, results) {
    console.log("createIndexAndStreamDataOnES function started", indexName);
    var elasticSearch = require('elasticsearch');
    var elasticSearchClient = new elasticSearch.Client({
        accessKeyId: 'your_keyid',
        secretAccessKey: 'your_secret',
        service: 'es',
        region: 'your_region',
        host: 'your_end_point'
    });
    elasticSearchClient.create({
        index: indexName,
        type: 'test',
        id: '1',
        body: {
            title: 'Test 1',
            tags: ['y', 'z'],
            published: true,
            published_at: '2013-01-01',
            counter: 1
        }
    }, function(err, data) {
        if (err) {
            console.log("Error on creating data", err);
            next(err);
        } else {
            console.log("data reply received", data);
            next(null, true);
        }
    });
}

Thanks